diff options
Diffstat (limited to 'mm/memcontrol.c')
| -rw-r--r-- | mm/memcontrol.c | 30 |
1 files changed, 19 insertions, 11 deletions
diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e25b93a4267d..8b421b246842 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -67,6 +67,8 @@ #include <net/sock.h> #include <net/ip.h> #include <net/tcp_memcontrol.h> +#include <linux/locallock.h> + #include "slab.h" #include <asm/uaccess.h> @@ -87,6 +89,7 @@ int do_swap_account __read_mostly; #define do_swap_account 0 #endif +static DEFINE_LOCAL_IRQ_LOCK(event_lock); static const char * const mem_cgroup_stat_names[] = { "cache", "rss", @@ -1922,14 +1925,17 @@ static void drain_local_stock(struct work_struct *dummy) */ static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) { - struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); + struct memcg_stock_pcp *stock; + int cpu = get_cpu_light(); + + stock = &per_cpu(memcg_stock, cpu); if (stock->cached != memcg) { /* reset if necessary */ drain_stock(stock); stock->cached = memcg; } stock->nr_pages += nr_pages; - put_cpu_var(memcg_stock); + put_cpu_light(); } /* @@ -1945,7 +1951,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg) return; /* Notify other cpus that system-wide "drain" is running */ get_online_cpus(); - curcpu = get_cpu(); + curcpu = get_cpu_light(); for_each_online_cpu(cpu) { struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); struct mem_cgroup *memcg; @@ -1962,7 +1968,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg) schedule_work_on(cpu, &stock->work); } } - put_cpu(); + put_cpu_light(); put_online_cpus(); mutex_unlock(&percpu_charge_mutex); } @@ -4691,12 +4697,12 @@ static int mem_cgroup_move_account(struct page *page, ret = 0; - local_irq_disable(); + local_lock_irq(event_lock); mem_cgroup_charge_statistics(to, page, nr_pages); memcg_check_events(to, page); mem_cgroup_charge_statistics(from, page, -nr_pages); memcg_check_events(from, page); - local_irq_enable(); + local_unlock_irq(event_lock); out_unlock: unlock_page(page); out: @@ -5486,10 +5492,10 @@ void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, VM_BUG_ON_PAGE(!PageTransHuge(page), page); } - local_irq_disable(); + local_lock_irq(event_lock); mem_cgroup_charge_statistics(memcg, page, nr_pages); memcg_check_events(memcg, page); - local_irq_enable(); + local_unlock_irq(event_lock); if (do_swap_account && PageSwapCache(page)) { swp_entry_t entry = { .val = page_private(page) }; @@ -5545,14 +5551,14 @@ static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout, memcg_oom_recover(memcg); } - local_irq_save(flags); + local_lock_irqsave(event_lock, flags); __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); __this_cpu_add(memcg->stat->nr_page_events, nr_pages); memcg_check_events(memcg, dummy_page); - local_irq_restore(flags); + local_unlock_irqrestore(event_lock, flags); if (!mem_cgroup_is_root(memcg)) css_put_many(&memcg->css, nr_pages); @@ -5762,6 +5768,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) { struct mem_cgroup *memcg, *swap_memcg; unsigned short oldid; + unsigned long flags; VM_BUG_ON_PAGE(PageLRU(page), page); VM_BUG_ON_PAGE(page_count(page), page); @@ -5802,9 +5809,10 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) * important here to have the interrupts disabled because it is the * only synchronisation we have for udpating the per-CPU variables. */ - VM_BUG_ON(!irqs_disabled()); + local_lock_irqsave(event_lock, flags); mem_cgroup_charge_statistics(memcg, page, -1); memcg_check_events(memcg, page); + local_unlock_irqrestore(event_lock, flags); if (!mem_cgroup_is_root(memcg)) css_put(&memcg->css); |