2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2011-05-27 09:28:01 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-09-22 16:43:50 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-21 17:01:26 -04:00
|
|
|
|
2006-01-09 20:52:17 -08:00
|
|
|
|
2009-01-06 07:20:54 -08:00
|
|
|
|
2009-06-08 19:50:45 -04:00
|
|
|
|
2011-05-22 18:54:21 +02:00
|
|
|
|
2010-10-25 14:41:59 -04:00
|
|
|
|
2011-03-23 16:43:25 -07:00
|
|
|
|
2011-05-27 09:28:01 -04:00
|
|
|
|
2011-03-22 22:23:41 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-05-27 09:28:01 -04:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
|
|
|
|
|
2011-04-21 18:19:44 -06:00
|
|
|
|
2011-03-22 22:23:41 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2011-03-22 22:23:41 +11:00
|
|
|
|
2011-04-21 18:19:44 -06:00
|
|
|
|
2011-03-22 22:23:41 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2006-03-26 01:37:24 -08:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
|
|
|
|
|
2011-04-05 23:51:48 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fs: use fast counters for vfs caches
percpu_counter library generates quite nasty code, so unless you need
to dynamically allocate counters or take fast approximate value, a
simple per cpu set of counters is much better.
The percpu_counter can never be made to work as well, because it has an
indirection from pointer to percpu memory, and it can't use direct
this_cpu_inc interfaces because it doesn't use static PER_CPU data, so
code will always be worse.
In the fastpath, it is the difference between this:
incl %gs:nr_dentry # nr_dentry
and this:
movl percpu_counter_batch(%rip), %edx # percpu_counter_batch,
movl $1, %esi #,
movq $nr_dentry, %rdi #,
call __percpu_counter_add # (plus I clobber registers)
__percpu_counter_add:
pushq %rbp #
movq %rsp, %rbp #,
subq $32, %rsp #,
movq %rbx, -24(%rbp) #,
movq %r12, -16(%rbp) #,
movq %r13, -8(%rbp) #,
movq %rdi, %rbx # fbc, fbc
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
incl -8124(%rax) # <variable>.preempt_count
movq 32(%rdi), %r12 # <variable>.counters, tcp_ptr__
#APP
# 78 "lib/percpu_counter.c" 1
add %gs:this_cpu_off, %r12 # this_cpu_off, tcp_ptr__
# 0 "" 2
#NO_APP
movslq (%r12),%r13 #* tcp_ptr__, tmp73
movslq %edx,%rax # batch, batch
addq %rsi, %r13 # amount, count
cmpq %rax, %r13 # batch, count
jge .L27 #,
negl %edx # tmp76
movslq %edx,%rdx # tmp76, tmp77
cmpq %rdx, %r13 # tmp77, count
jg .L28 #,
.L27:
movq %rbx, %rdi # fbc,
call _raw_spin_lock #
addq %r13, 8(%rbx) # count, <variable>.count
movq %rbx, %rdi # fbc,
movl $0, (%r12) #,* tcp_ptr__
call _raw_spin_unlock #
.L29:
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
decl -8124(%rax) # <variable>.preempt_count
movq -8136(%rax), %rax #, D.14625
testb $8, %al #, D.14625
jne .L32 #,
.L31:
movq -24(%rbp), %rbx #,
movq -16(%rbp), %r12 #,
movq -8(%rbp), %r13 #,
leave
ret
.p2align 4,,10
.p2align 3
.L28:
movl %r13d, (%r12) # count,*
jmp .L29 #
.L32:
call preempt_schedule #
.p2align 4,,6
jmp .L31 #
.size __percpu_counter_add, .-__percpu_counter_add
.p2align 4,,15
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
2011-01-07 17:49:19 +11:00
|
|
|
|
2011-07-08 14:14:38 +10:00
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
fs: use fast counters for vfs caches
percpu_counter library generates quite nasty code, so unless you need
to dynamically allocate counters or take fast approximate value, a
simple per cpu set of counters is much better.
The percpu_counter can never be made to work as well, because it has an
indirection from pointer to percpu memory, and it can't use direct
this_cpu_inc interfaces because it doesn't use static PER_CPU data, so
code will always be worse.
In the fastpath, it is the difference between this:
incl %gs:nr_dentry # nr_dentry
and this:
movl percpu_counter_batch(%rip), %edx # percpu_counter_batch,
movl $1, %esi #,
movq $nr_dentry, %rdi #,
call __percpu_counter_add # (plus I clobber registers)
__percpu_counter_add:
pushq %rbp #
movq %rsp, %rbp #,
subq $32, %rsp #,
movq %rbx, -24(%rbp) #,
movq %r12, -16(%rbp) #,
movq %r13, -8(%rbp) #,
movq %rdi, %rbx # fbc, fbc
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
incl -8124(%rax) # <variable>.preempt_count
movq 32(%rdi), %r12 # <variable>.counters, tcp_ptr__
#APP
# 78 "lib/percpu_counter.c" 1
add %gs:this_cpu_off, %r12 # this_cpu_off, tcp_ptr__
# 0 "" 2
#NO_APP
movslq (%r12),%r13 #* tcp_ptr__, tmp73
movslq %edx,%rax # batch, batch
addq %rsi, %r13 # amount, count
cmpq %rax, %r13 # batch, count
jge .L27 #,
negl %edx # tmp76
movslq %edx,%rdx # tmp76, tmp77
cmpq %rdx, %r13 # tmp77, count
jg .L28 #,
.L27:
movq %rbx, %rdi # fbc,
call _raw_spin_lock #
addq %r13, 8(%rbx) # count, <variable>.count
movq %rbx, %rdi # fbc,
movl $0, (%r12) #,* tcp_ptr__
call _raw_spin_unlock #
.L29:
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
decl -8124(%rax) # <variable>.preempt_count
movq -8136(%rax), %rax #, D.14625
testb $8, %al #, D.14625
jne .L32 #,
.L31:
movq -24(%rbp), %rbx #,
movq -16(%rbp), %r12 #,
movq -8(%rbp), %r13 #,
leave
ret
.p2align 4,,10
.p2align 3
.L28:
movl %r13d, (%r12) # count,*
jmp .L29 #
.L32:
call preempt_schedule #
.p2align 4,,6
jmp .L31 #
.size __percpu_counter_add, .-__percpu_counter_add
.p2align 4,,15
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
2011-01-07 17:49:19 +11:00
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
fs: use fast counters for vfs caches
percpu_counter library generates quite nasty code, so unless you need
to dynamically allocate counters or take fast approximate value, a
simple per cpu set of counters is much better.
The percpu_counter can never be made to work as well, because it has an
indirection from pointer to percpu memory, and it can't use direct
this_cpu_inc interfaces because it doesn't use static PER_CPU data, so
code will always be worse.
In the fastpath, it is the difference between this:
incl %gs:nr_dentry # nr_dentry
and this:
movl percpu_counter_batch(%rip), %edx # percpu_counter_batch,
movl $1, %esi #,
movq $nr_dentry, %rdi #,
call __percpu_counter_add # (plus I clobber registers)
__percpu_counter_add:
pushq %rbp #
movq %rsp, %rbp #,
subq $32, %rsp #,
movq %rbx, -24(%rbp) #,
movq %r12, -16(%rbp) #,
movq %r13, -8(%rbp) #,
movq %rdi, %rbx # fbc, fbc
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
incl -8124(%rax) # <variable>.preempt_count
movq 32(%rdi), %r12 # <variable>.counters, tcp_ptr__
#APP
# 78 "lib/percpu_counter.c" 1
add %gs:this_cpu_off, %r12 # this_cpu_off, tcp_ptr__
# 0 "" 2
#NO_APP
movslq (%r12),%r13 #* tcp_ptr__, tmp73
movslq %edx,%rax # batch, batch
addq %rsi, %r13 # amount, count
cmpq %rax, %r13 # batch, count
jge .L27 #,
negl %edx # tmp76
movslq %edx,%rdx # tmp76, tmp77
cmpq %rdx, %r13 # tmp77, count
jg .L28 #,
.L27:
movq %rbx, %rdi # fbc,
call _raw_spin_lock #
addq %r13, 8(%rbx) # count, <variable>.count
movq %rbx, %rdi # fbc,
movl $0, (%r12) #,* tcp_ptr__
call _raw_spin_unlock #
.L29:
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
decl -8124(%rax) # <variable>.preempt_count
movq -8136(%rax), %rax #, D.14625
testb $8, %al #, D.14625
jne .L32 #,
.L31:
movq -24(%rbp), %rbx #,
movq -16(%rbp), %r12 #,
movq -8(%rbp), %r13 #,
leave
ret
.p2align 4,,10
.p2align 3
.L28:
movl %r13d, (%r12) # count,*
jmp .L29 #
.L32:
call preempt_schedule #
.p2align 4,,6
jmp .L31 #
.size __percpu_counter_add, .-__percpu_counter_add
.p2align 4,,15
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
2011-01-07 17:49:19 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:38 +10:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fs: use fast counters for vfs caches
percpu_counter library generates quite nasty code, so unless you need
to dynamically allocate counters or take fast approximate value, a
simple per cpu set of counters is much better.
The percpu_counter can never be made to work as well, because it has an
indirection from pointer to percpu memory, and it can't use direct
this_cpu_inc interfaces because it doesn't use static PER_CPU data, so
code will always be worse.
In the fastpath, it is the difference between this:
incl %gs:nr_dentry # nr_dentry
and this:
movl percpu_counter_batch(%rip), %edx # percpu_counter_batch,
movl $1, %esi #,
movq $nr_dentry, %rdi #,
call __percpu_counter_add # (plus I clobber registers)
__percpu_counter_add:
pushq %rbp #
movq %rsp, %rbp #,
subq $32, %rsp #,
movq %rbx, -24(%rbp) #,
movq %r12, -16(%rbp) #,
movq %r13, -8(%rbp) #,
movq %rdi, %rbx # fbc, fbc
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
incl -8124(%rax) # <variable>.preempt_count
movq 32(%rdi), %r12 # <variable>.counters, tcp_ptr__
#APP
# 78 "lib/percpu_counter.c" 1
add %gs:this_cpu_off, %r12 # this_cpu_off, tcp_ptr__
# 0 "" 2
#NO_APP
movslq (%r12),%r13 #* tcp_ptr__, tmp73
movslq %edx,%rax # batch, batch
addq %rsi, %r13 # amount, count
cmpq %rax, %r13 # batch, count
jge .L27 #,
negl %edx # tmp76
movslq %edx,%rdx # tmp76, tmp77
cmpq %rdx, %r13 # tmp77, count
jg .L28 #,
.L27:
movq %rbx, %rdi # fbc,
call _raw_spin_lock #
addq %r13, 8(%rbx) # count, <variable>.count
movq %rbx, %rdi # fbc,
movl $0, (%r12) #,* tcp_ptr__
call _raw_spin_unlock #
.L29:
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
decl -8124(%rax) # <variable>.preempt_count
movq -8136(%rax), %rax #, D.14625
testb $8, %al #, D.14625
jne .L32 #,
.L31:
movq -24(%rbp), %rbx #,
movq -16(%rbp), %r12 #,
movq -8(%rbp), %r13 #,
leave
ret
.p2align 4,,10
.p2align 3
.L28:
movl %r13d, (%r12) # count,*
jmp .L29 #
.L32:
call preempt_schedule #
.p2align 4,,6
jmp .L31 #
.size __percpu_counter_add, .-__percpu_counter_add
.p2align 4,,15
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
2011-01-07 17:49:19 +11:00
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:38 +10:00
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
2009-01-06 14:41:13 -08:00
|
|
|
|
|
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-08-07 14:38:25 -03:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-09-21 17:01:11 -07:00
|
|
|
|
2006-03-28 01:56:41 -08:00
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-10-28 14:13:30 +02:00
|
|
|
|
2011-08-06 22:45:50 -07:00
|
|
|
|
2008-12-09 09:34:39 -05:00
|
|
|
|
|
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-02-04 09:06:57 -05:00
|
|
|
|
|
|
|
|
|
2009-08-07 14:38:25 -03:00
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-06-24 14:29:43 -04:00
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-01-06 14:39:23 -08:00
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-09-15 09:43:56 +02:00
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-06-08 19:50:45 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
2009-05-21 17:01:26 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fs: use fast counters for vfs caches
percpu_counter library generates quite nasty code, so unless you need
to dynamically allocate counters or take fast approximate value, a
simple per cpu set of counters is much better.
The percpu_counter can never be made to work as well, because it has an
indirection from pointer to percpu memory, and it can't use direct
this_cpu_inc interfaces because it doesn't use static PER_CPU data, so
code will always be worse.
In the fastpath, it is the difference between this:
incl %gs:nr_dentry # nr_dentry
and this:
movl percpu_counter_batch(%rip), %edx # percpu_counter_batch,
movl $1, %esi #,
movq $nr_dentry, %rdi #,
call __percpu_counter_add # (plus I clobber registers)
__percpu_counter_add:
pushq %rbp #
movq %rsp, %rbp #,
subq $32, %rsp #,
movq %rbx, -24(%rbp) #,
movq %r12, -16(%rbp) #,
movq %r13, -8(%rbp) #,
movq %rdi, %rbx # fbc, fbc
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
incl -8124(%rax) # <variable>.preempt_count
movq 32(%rdi), %r12 # <variable>.counters, tcp_ptr__
#APP
# 78 "lib/percpu_counter.c" 1
add %gs:this_cpu_off, %r12 # this_cpu_off, tcp_ptr__
# 0 "" 2
#NO_APP
movslq (%r12),%r13 #* tcp_ptr__, tmp73
movslq %edx,%rax # batch, batch
addq %rsi, %r13 # amount, count
cmpq %rax, %r13 # batch, count
jge .L27 #,
negl %edx # tmp76
movslq %edx,%rdx # tmp76, tmp77
cmpq %rdx, %r13 # tmp77, count
jg .L28 #,
.L27:
movq %rbx, %rdi # fbc,
call _raw_spin_lock #
addq %r13, 8(%rbx) # count, <variable>.count
movq %rbx, %rdi # fbc,
movl $0, (%r12) #,* tcp_ptr__
call _raw_spin_unlock #
.L29:
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
decl -8124(%rax) # <variable>.preempt_count
movq -8136(%rax), %rax #, D.14625
testb $8, %al #, D.14625
jne .L32 #,
.L31:
movq -24(%rbp), %rbx #,
movq -16(%rbp), %r12 #,
movq -8(%rbp), %r13 #,
leave
ret
.p2align 4,,10
.p2align 3
.L28:
movl %r13d, (%r12) # count,*
jmp .L29 #
.L32:
call preempt_schedule #
.p2align 4,,6
jmp .L31 #
.size __percpu_counter_add, .-__percpu_counter_add
.p2align 4,,15
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
2011-01-07 17:49:19 +11:00
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
2009-08-07 14:38:25 -03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-08-07 14:38:25 -03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-30 17:32:23 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-01-07 17:49:50 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-08-07 14:38:29 -03:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-04-02 13:38:18 +02:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-05-21 17:01:26 -04:00
|
|
|
|
2009-06-08 19:50:45 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fs: use fast counters for vfs caches
percpu_counter library generates quite nasty code, so unless you need
to dynamically allocate counters or take fast approximate value, a
simple per cpu set of counters is much better.
The percpu_counter can never be made to work as well, because it has an
indirection from pointer to percpu memory, and it can't use direct
this_cpu_inc interfaces because it doesn't use static PER_CPU data, so
code will always be worse.
In the fastpath, it is the difference between this:
incl %gs:nr_dentry # nr_dentry
and this:
movl percpu_counter_batch(%rip), %edx # percpu_counter_batch,
movl $1, %esi #,
movq $nr_dentry, %rdi #,
call __percpu_counter_add # (plus I clobber registers)
__percpu_counter_add:
pushq %rbp #
movq %rsp, %rbp #,
subq $32, %rsp #,
movq %rbx, -24(%rbp) #,
movq %r12, -16(%rbp) #,
movq %r13, -8(%rbp) #,
movq %rdi, %rbx # fbc, fbc
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
incl -8124(%rax) # <variable>.preempt_count
movq 32(%rdi), %r12 # <variable>.counters, tcp_ptr__
#APP
# 78 "lib/percpu_counter.c" 1
add %gs:this_cpu_off, %r12 # this_cpu_off, tcp_ptr__
# 0 "" 2
#NO_APP
movslq (%r12),%r13 #* tcp_ptr__, tmp73
movslq %edx,%rax # batch, batch
addq %rsi, %r13 # amount, count
cmpq %rax, %r13 # batch, count
jge .L27 #,
negl %edx # tmp76
movslq %edx,%rdx # tmp76, tmp77
cmpq %rdx, %r13 # tmp77, count
jg .L28 #,
.L27:
movq %rbx, %rdi # fbc,
call _raw_spin_lock #
addq %r13, 8(%rbx) # count, <variable>.count
movq %rbx, %rdi # fbc,
movl $0, (%r12) #,* tcp_ptr__
call _raw_spin_unlock #
.L29:
#APP
# 216 "/home/npiggin/usr/src/linux-2.6/arch/x86/include/asm/thread_info.h" 1
movq %gs:kernel_stack,%rax #, pfo_ret__
# 0 "" 2
#NO_APP
decl -8124(%rax) # <variable>.preempt_count
movq -8136(%rax), %rax #, D.14625
testb $8, %al #, D.14625
jne .L32 #,
.L31:
movq -24(%rbp), %rbx #,
movq -16(%rbp), %r12 #,
movq -8(%rbp), %r13 #,
leave
ret
.p2align 4,,10
.p2align 3
.L28:
movl %r13d, (%r12) # count,*
jmp .L29 #
.L32:
call preempt_schedule #
.p2align 4,,6
jmp .L31 #
.size __percpu_counter_add, .-__percpu_counter_add
.p2align 4,,15
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
2011-01-07 17:49:19 +11:00
|
|
|
|
2009-08-07 14:38:29 -03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-07 17:49:49 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-06 10:48:55 +02:00
|
|
|
|
2009-08-07 14:38:29 -03:00
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
2009-08-07 14:38:29 -03:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-07 17:49:49 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2011-02-23 13:49:47 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-05-24 17:12:06 -07:00
|
|
|
|
2011-02-23 13:49:47 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
|
|
|
|
|
2011-02-23 13:49:47 +01:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-05-21 17:01:26 -04:00
|
|
|
|
2009-12-17 21:24:24 -05:00
|
|
|
|
2009-05-21 17:01:26 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-07-25 19:45:34 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2007-05-16 22:10:57 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
|
|
|
|
|
2010-05-14 10:49:22 +01:00
|
|
|
|
2010-10-23 11:11:40 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:38 +10:00
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
2010-05-14 10:49:22 +01:00
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2011-07-08 14:14:38 +10:00
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
2011-07-26 11:36:34 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
|
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-05-27 09:28:01 -04:00
|
|
|
|
|
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-28 06:41:09 +02:00
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-28 06:41:09 +02:00
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
2011-07-28 06:41:09 +02:00
|
|
|
|
2010-10-23 06:58:09 -04:00
|
|
|
|
2010-06-04 20:55:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-06-27 16:18:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-06-04 20:55:25 -04:00
|
|
|
|
2011-06-27 16:18:10 -07:00
|
|
|
|
2010-06-04 20:55:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-01-07 17:49:49 +11:00
|
|
|
|
2010-06-04 20:55:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:37 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-06-07 13:21:05 -04:00
|
|
|
|
2010-06-04 19:33:20 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:37 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-28 06:11:47 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
|
|
|
|
|
2010-06-04 19:40:39 -04:00
|
|
|
|
|
|
|
|
|
2010-06-04 19:33:20 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-06-07 13:23:20 -04:00
|
|
|
|
2010-06-04 19:33:20 -04:00
|
|
|
|
2010-06-04 20:19:55 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:37 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-06-04 19:33:20 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-06-07 13:21:05 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-25 20:49:35 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-25 20:49:35 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-25 20:49:35 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2010-10-25 20:49:35 -04:00
|
|
|
|
|
|
|
|
|
2009-03-11 13:17:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2010-10-25 20:49:35 -04:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2010-10-25 20:49:35 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-24 19:40:33 +02:00
|
|
|
|
|
|
|
|
|
2011-02-24 17:25:47 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-24 19:40:33 +02:00
|
|
|
|
|
|
|
|
|
2011-02-24 17:25:47 +11:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-02-24 17:25:47 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 05:03:02 -04:00
|
|
|
|
2010-10-24 19:40:33 +02:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2010-10-24 19:40:33 +02:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-11 13:17:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-02-24 17:25:47 +11:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-02-24 17:25:47 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 19:07:20 +02:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2010-10-23 19:07:20 +02:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2010-10-23 19:07:20 +02:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-24 19:40:33 +02:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:42 +10:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-07-08 14:14:42 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2011-07-08 14:14:42 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-10-28 10:03:41 +02:00
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
2011-03-22 22:23:39 +11:00
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2011-07-08 14:14:38 +10:00
|
|
|
|
2010-10-23 06:55:17 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2011-03-22 22:23:39 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2007-02-10 01:45:39 -08:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:38 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
fs: new inode i_state corruption fix
There was a report of a data corruption
http://lkml.org/lkml/2008/11/14/121. There is a script included to
reproduce the problem.
During testing, I encountered a number of strange things with ext3, so I
tried ext2 to attempt to reduce complexity of the problem. I found that
fsstress would quickly hang in wait_on_inode, waiting for I_LOCK to be
cleared, even though instrumentation showed that unlock_new_inode had
already been called for that inode. This points to memory scribble, or
synchronisation problme.
i_state of I_NEW inodes is not protected by inode_lock because other
processes are not supposed to touch them until I_LOCK (and I_NEW) is
cleared. Adding WARN_ON(inode->i_state & I_NEW) to sites where we modify
i_state revealed that generic_sync_sb_inodes is picking up new inodes from
the inode lists and passing them to __writeback_single_inode without
waiting for I_NEW. Subsequently modifying i_state causes corruption. In
my case it would look like this:
CPU0 CPU1
unlock_new_inode() __sync_single_inode()
reg <- inode->i_state
reg -> reg & ~(I_LOCK|I_NEW) reg <- inode->i_state
reg -> inode->i_state reg -> reg | I_SYNC
reg -> inode->i_state
Non-atomic RMW on CPU1 overwrites CPU0 store and sets I_LOCK|I_NEW again.
Fix for this is rather than wait for I_NEW inodes, just skip over them:
inodes concurrently being created are not subject to data integrity
operations, and should not significantly contribute to dirty memory
either.
After this change, I'm unable to reproduce any of the added warnings or
hangs after ~1hour of running. Previously, the new warnings would start
immediately and hang would happen in under 5 minutes.
I'm also testing on ext3 now, and so far no problems there either. I
don't know whether this fixes the problem reported above, but it fixes a
real problem for me.
Cc: "Jorge Boncompte [DTI2]" <jorge@dti2.net>
Reported-by: Adrian Hunter <ext-adrian.hunter@nokia.com>
Cc: Jan Kara <jack@suse.cz>
Cc: <stable@kernel.org>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-03-12 14:31:38 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:39 +10:00
|
|
|
|
2011-07-08 14:14:38 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-06-30 01:55:45 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-07-08 14:14:40 +10:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2008-04-29 00:59:40 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2010-06-02 17:38:30 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 07:09:06 -04:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2010-10-23 07:09:06 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 07:09:06 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2008-04-29 00:59:40 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2010-06-02 17:38:30 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 07:09:06 -04:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2010-10-23 07:09:06 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 07:09:06 -04:00
|
|
|
|
2008-10-30 17:35:24 +11:00
|
|
|
|
|
|
|
|
|
2010-10-23 11:18:01 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-30 17:35:24 +11:00
|
|
|
|
2010-10-23 11:18:01 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-30 17:35:24 +11:00
|
|
|
|
2010-10-23 11:18:01 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 11:19:54 -04:00
|
|
|
|
2008-10-30 17:35:24 +11:00
|
|
|
|
2010-10-23 11:18:01 -04:00
|
|
|
|
|
|
|
|
|
2008-10-30 17:35:24 +11:00
|
|
|
|
2010-10-23 11:18:01 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-10-30 17:35:24 +11:00
|
|
|
|
2010-10-23 11:19:54 -04:00
|
|
|
|
2008-10-30 17:35:24 +11:00
|
|
|
|
2011-07-26 11:36:34 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-17 04:03:05 -07:00
|
|
|
|
2009-01-06 14:39:23 -08:00
|
|
|
|
2007-07-17 04:03:05 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2011-07-26 11:36:34 +02:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-14 01:38:33 +02:00
|
|
|
|
lockdep: Add helper function for dir vs file i_mutex annotation
Purely in-memory filesystems do not use the inode hash as the dcache
tells us if an entry already exists. As a result, they do not call
unlock_new_inode, and thus directory inodes do not get put into a
different lockdep class for i_sem.
We need the different lockdep classes, because the locking order for
i_mutex is different for directory inodes and regular inodes. Directory
inodes can do "readdir()", which takes i_mutex *before* possibly taking
mm->mmap_sem (due to a page fault while copying the directory entry to
user space).
In contrast, regular inodes can be mmap'ed, which takes mm->mmap_sem
before accessing i_mutex.
The two cases can never happen for the same inode, so no real deadlock
can occur, but without the different lockdep classes, lockdep cannot
understand that. As a result, if CONFIG_DEBUG_LOCK_ALLOC is set, this
can lead to false positives from lockdep like below:
find/645 is trying to acquire lock:
(&mm->mmap_sem){++++++}, at: [<ffffffff81109514>] might_fault+0x5c/0xac
but task is already holding lock:
(&sb->s_type->i_mutex_key#15){+.+.+.}, at: [<ffffffff81149f34>]
vfs_readdir+0x5b/0xb4
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (&sb->s_type->i_mutex_key#15){+.+.+.}:
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff814db822>] __mutex_lock_common+0x4c/0x361
[<ffffffff814dbc46>] mutex_lock_nested+0x40/0x45
[<ffffffff811daa87>] hugetlbfs_file_mmap+0x82/0x110
[<ffffffff81111557>] mmap_region+0x258/0x432
[<ffffffff811119dd>] do_mmap_pgoff+0x2ac/0x306
[<ffffffff81111b4f>] sys_mmap_pgoff+0x118/0x16a
[<ffffffff8100c858>] sys_mmap+0x22/0x24
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
-> #0 (&mm->mmap_sem){++++++}:
[<ffffffff8108a4bc>] __lock_acquire+0xa1a/0xcf7
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff81109541>] might_fault+0x89/0xac
[<ffffffff81149cff>] filldir+0x6f/0xc7
[<ffffffff811586ea>] dcache_readdir+0x67/0x205
[<ffffffff81149f54>] vfs_readdir+0x7b/0xb4
[<ffffffff8114a073>] sys_getdents+0x7e/0xd1
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
This patch moves the directory vs file lockdep annotation into a helper
function that can be called by in-memory filesystems and has hugetlbfs
call it.
Signed-off-by: Josh Boyer <jwboyer@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-08-25 07:48:12 -04:00
|
|
|
|
|
|
|
|
|
2010-10-11 22:38:00 +09:00
|
|
|
|
2007-10-16 06:47:54 +02:00
|
|
|
|
|
|
|
|
|
2009-06-04 15:26:49 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-16 06:47:54 +02:00
|
|
|
|
lockdep: Add helper function for dir vs file i_mutex annotation
Purely in-memory filesystems do not use the inode hash as the dcache
tells us if an entry already exists. As a result, they do not call
unlock_new_inode, and thus directory inodes do not get put into a
different lockdep class for i_sem.
We need the different lockdep classes, because the locking order for
i_mutex is different for directory inodes and regular inodes. Directory
inodes can do "readdir()", which takes i_mutex *before* possibly taking
mm->mmap_sem (due to a page fault while copying the directory entry to
user space).
In contrast, regular inodes can be mmap'ed, which takes mm->mmap_sem
before accessing i_mutex.
The two cases can never happen for the same inode, so no real deadlock
can occur, but without the different lockdep classes, lockdep cannot
understand that. As a result, if CONFIG_DEBUG_LOCK_ALLOC is set, this
can lead to false positives from lockdep like below:
find/645 is trying to acquire lock:
(&mm->mmap_sem){++++++}, at: [<ffffffff81109514>] might_fault+0x5c/0xac
but task is already holding lock:
(&sb->s_type->i_mutex_key#15){+.+.+.}, at: [<ffffffff81149f34>]
vfs_readdir+0x5b/0xb4
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (&sb->s_type->i_mutex_key#15){+.+.+.}:
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff814db822>] __mutex_lock_common+0x4c/0x361
[<ffffffff814dbc46>] mutex_lock_nested+0x40/0x45
[<ffffffff811daa87>] hugetlbfs_file_mmap+0x82/0x110
[<ffffffff81111557>] mmap_region+0x258/0x432
[<ffffffff811119dd>] do_mmap_pgoff+0x2ac/0x306
[<ffffffff81111b4f>] sys_mmap_pgoff+0x118/0x16a
[<ffffffff8100c858>] sys_mmap+0x22/0x24
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
-> #0 (&mm->mmap_sem){++++++}:
[<ffffffff8108a4bc>] __lock_acquire+0xa1a/0xcf7
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff81109541>] might_fault+0x89/0xac
[<ffffffff81149cff>] filldir+0x6f/0xc7
[<ffffffff811586ea>] dcache_readdir+0x67/0x205
[<ffffffff81149f54>] vfs_readdir+0x7b/0xb4
[<ffffffff8114a073>] sys_getdents+0x7e/0xd1
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
This patch moves the directory vs file lockdep annotation into a helper
function that can be called by in-memory filesystems and has hugetlbfs
call it.
Signed-off-by: Josh Boyer <jwboyer@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-08-25 07:48:12 -04:00
|
|
|
|
|
|
|
|
|
2007-10-14 01:38:33 +02:00
|
|
|
|
lockdep: Add helper function for dir vs file i_mutex annotation
Purely in-memory filesystems do not use the inode hash as the dcache
tells us if an entry already exists. As a result, they do not call
unlock_new_inode, and thus directory inodes do not get put into a
different lockdep class for i_sem.
We need the different lockdep classes, because the locking order for
i_mutex is different for directory inodes and regular inodes. Directory
inodes can do "readdir()", which takes i_mutex *before* possibly taking
mm->mmap_sem (due to a page fault while copying the directory entry to
user space).
In contrast, regular inodes can be mmap'ed, which takes mm->mmap_sem
before accessing i_mutex.
The two cases can never happen for the same inode, so no real deadlock
can occur, but without the different lockdep classes, lockdep cannot
understand that. As a result, if CONFIG_DEBUG_LOCK_ALLOC is set, this
can lead to false positives from lockdep like below:
find/645 is trying to acquire lock:
(&mm->mmap_sem){++++++}, at: [<ffffffff81109514>] might_fault+0x5c/0xac
but task is already holding lock:
(&sb->s_type->i_mutex_key#15){+.+.+.}, at: [<ffffffff81149f34>]
vfs_readdir+0x5b/0xb4
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #1 (&sb->s_type->i_mutex_key#15){+.+.+.}:
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff814db822>] __mutex_lock_common+0x4c/0x361
[<ffffffff814dbc46>] mutex_lock_nested+0x40/0x45
[<ffffffff811daa87>] hugetlbfs_file_mmap+0x82/0x110
[<ffffffff81111557>] mmap_region+0x258/0x432
[<ffffffff811119dd>] do_mmap_pgoff+0x2ac/0x306
[<ffffffff81111b4f>] sys_mmap_pgoff+0x118/0x16a
[<ffffffff8100c858>] sys_mmap+0x22/0x24
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
-> #0 (&mm->mmap_sem){++++++}:
[<ffffffff8108a4bc>] __lock_acquire+0xa1a/0xcf7
[<ffffffff8108ac26>] lock_acquire+0xbf/0x103
[<ffffffff81109541>] might_fault+0x89/0xac
[<ffffffff81149cff>] filldir+0x6f/0xc7
[<ffffffff811586ea>] dcache_readdir+0x67/0x205
[<ffffffff81149f54>] vfs_readdir+0x7b/0xb4
[<ffffffff8114a073>] sys_getdents+0x7e/0xd1
[<ffffffff814e3ec2>] system_call_fastpath+0x16/0x1b
This patch moves the directory vs file lockdep annotation into a helper
function that can be called by in-memory filesystems and has hugetlbfs
call it.
Signed-off-by: Josh Boyer <jwboyer@redhat.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-08-25 07:48:12 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2009-12-17 14:25:01 +01:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2010-10-23 07:15:32 -04:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:40 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
|
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
inode numbering: make static counters in new_inode and iunique be 32 bits
The problems are:
- on filesystems w/o permanent inode numbers, i_ino values can be larger
than 32 bits, which can cause problems for some 32 bit userspace programs on
a 64 bit kernel. We can't do anything for filesystems that have actual
>32-bit inode numbers, but on filesystems that generate i_ino values on the
fly, we should try to have them fit in 32 bits. We could trivially fix this
by making the static counters in new_inode and iunique 32 bits, but...
- many filesystems call new_inode and assume that the i_ino values they are
given are unique. They are not guaranteed to be so, since the static
counter can wrap. This problem is exacerbated by the fix for #1.
- after allocating a new inode, some filesystems call iunique to try to get
a unique i_ino value, but they don't actually add their inodes to the
hashtable, and so they're still not guaranteed to be unique if that counter
wraps.
This patch set takes the simpler approach of simply using iunique and hashing
the inodes afterward. Christoph H. previously mentioned that he thought that
this approach may slow down lookups for filesystems that currently hash their
inodes.
The questions are:
1) how much would this slow down lookups for these filesystems?
2) is it enough to justify adding more infrastructure to avoid it?
What might be best is to start with this approach and then only move to using
IDR or some other scheme if these extra inodes in the hashtable prove to be
problematic.
I've done some cursory testing with this patch and the overhead of hashing and
unhashing the inodes with pipefs is pretty low -- just a few seconds of system
time added on to the creation and destruction of 10 million pipes (very
similar to the overhead that the IDR approach would add).
The hard thing to measure is what effect this has on other filesystems. I'm
open to ways to try and gauge this.
Again, I've only converted pipefs as an example. If this approach is
acceptable then I'll start work on patches to convert other filesystems.
With a pretty-much-worst-case microbenchmark provided by Eric Dumazet
<dada1@cosmosbay.com>:
hashing patch (pipebench):
sys 1m15.329s
sys 1m16.249s
sys 1m17.169s
unpatched (pipebench):
sys 1m9.836s
sys 1m12.541s
sys 1m14.153s
Which works out to 1.05642174294555027017. So ~5-6% slowdown.
This patch:
When a 32-bit program that was not compiled with large file offsets does a
stat and gets a st_ino value back that won't fit in the 32 bit field, glibc
(correctly) generates an EOVERFLOW error. We can't do anything about fs's
with larger permanent inode numbers, but when we generate them on the fly, we
ought to try and have them fit within a 32 bit field.
This patch takes the first step toward this by making the static counters in
these two functions be 32 bits.
[jlayton@redhat.com: mention that it's only the case for 32bit, non-LFS stat]
Signed-off-by: Jeff Layton <jlayton@redhat.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 00:32:29 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
inode numbering: make static counters in new_inode and iunique be 32 bits
The problems are:
- on filesystems w/o permanent inode numbers, i_ino values can be larger
than 32 bits, which can cause problems for some 32 bit userspace programs on
a 64 bit kernel. We can't do anything for filesystems that have actual
>32-bit inode numbers, but on filesystems that generate i_ino values on the
fly, we should try to have them fit in 32 bits. We could trivially fix this
by making the static counters in new_inode and iunique 32 bits, but...
- many filesystems call new_inode and assume that the i_ino values they are
given are unique. They are not guaranteed to be so, since the static
counter can wrap. This problem is exacerbated by the fix for #1.
- after allocating a new inode, some filesystems call iunique to try to get
a unique i_ino value, but they don't actually add their inodes to the
hashtable, and so they're still not guaranteed to be unique if that counter
wraps.
This patch set takes the simpler approach of simply using iunique and hashing
the inodes afterward. Christoph H. previously mentioned that he thought that
this approach may slow down lookups for filesystems that currently hash their
inodes.
The questions are:
1) how much would this slow down lookups for these filesystems?
2) is it enough to justify adding more infrastructure to avoid it?
What might be best is to start with this approach and then only move to using
IDR or some other scheme if these extra inodes in the hashtable prove to be
problematic.
I've done some cursory testing with this patch and the overhead of hashing and
unhashing the inodes with pipefs is pretty low -- just a few seconds of system
time added on to the creation and destruction of 10 million pipes (very
similar to the overhead that the IDR approach would add).
The hard thing to measure is what effect this has on other filesystems. I'm
open to ways to try and gauge this.
Again, I've only converted pipefs as an example. If this approach is
acceptable then I'll start work on patches to convert other filesystems.
With a pretty-much-worst-case microbenchmark provided by Eric Dumazet
<dada1@cosmosbay.com>:
hashing patch (pipebench):
sys 1m15.329s
sys 1m16.249s
sys 1m17.169s
unpatched (pipebench):
sys 1m9.836s
sys 1m12.541s
sys 1m14.153s
Which works out to 1.05642174294555027017. So ~5-6% slowdown.
This patch:
When a 32-bit program that was not compiled with large file offsets does a
stat and gets a st_ino value back that won't fit in the 32 bit field, glibc
(correctly) generates an EOVERFLOW error. We can't do anything about fs's
with larger permanent inode numbers, but when we generate them on the fly, we
ought to try and have them fit within a 32 bit field.
This patch takes the first step toward this by making the static counters in
these two functions be 32 bits.
[jlayton@redhat.com: mention that it's only the case for 32bit, non-LFS stat]
Signed-off-by: Jeff Layton <jlayton@redhat.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-08 00:32:29 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2007-05-08 00:29:48 -07:00
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
2007-05-08 00:29:48 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 07:00:16 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2007-05-08 00:29:48 -07:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-26 13:27:47 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-07-13 01:10:44 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-07-13 01:10:44 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-07-13 01:10:44 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-23 15:03:28 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
2010-10-23 15:19:20 -04:00
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2009-05-13 19:13:40 +01:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
2010-10-23 15:19:20 -04:00
|
|
|
|
2008-12-30 01:48:21 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-23 15:19:20 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-09-18 13:05:44 -07:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-09-18 13:05:44 -07:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2011-07-07 15:45:59 -04:00
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:37 +11:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-06-07 13:43:19 -04:00
|
|
|
|
2005-06-23 00:09:01 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
fs: new inode i_state corruption fix
There was a report of a data corruption
http://lkml.org/lkml/2008/11/14/121. There is a script included to
reproduce the problem.
During testing, I encountered a number of strange things with ext3, so I
tried ext2 to attempt to reduce complexity of the problem. I found that
fsstress would quickly hang in wait_on_inode, waiting for I_LOCK to be
cleared, even though instrumentation showed that unlock_new_inode had
already been called for that inode. This points to memory scribble, or
synchronisation problme.
i_state of I_NEW inodes is not protected by inode_lock because other
processes are not supposed to touch them until I_LOCK (and I_NEW) is
cleared. Adding WARN_ON(inode->i_state & I_NEW) to sites where we modify
i_state revealed that generic_sync_sb_inodes is picking up new inodes from
the inode lists and passing them to __writeback_single_inode without
waiting for I_NEW. Subsequently modifying i_state causes corruption. In
my case it would look like this:
CPU0 CPU1
unlock_new_inode() __sync_single_inode()
reg <- inode->i_state
reg -> reg & ~(I_LOCK|I_NEW) reg <- inode->i_state
reg -> inode->i_state reg -> reg | I_SYNC
reg -> inode->i_state
Non-atomic RMW on CPU1 overwrites CPU0 store and sets I_LOCK|I_NEW again.
Fix for this is rather than wait for I_NEW inodes, just skip over them:
inodes concurrently being created are not subject to data integrity
operations, and should not significantly contribute to dirty memory
either.
After this change, I'm unable to reproduce any of the added warnings or
hangs after ~1hour of running. Previously, the new warnings would start
immediately and hang would happen in under 5 minutes.
I'm also testing on ext3 now, and so far no problems there either. I
don't know whether this fixes the problem reported above, but it fixes a
real problem for me.
Cc: "Jorge Boncompte [DTI2]" <jorge@dti2.net>
Reported-by: Adrian Hunter <ext-adrian.hunter@nokia.com>
Cc: Jan Kara <jack@suse.cz>
Cc: <stable@kernel.org>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-03-12 14:31:38 -07:00
|
|
|
|
2005-06-23 00:09:01 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2010-10-21 11:49:30 +11:00
|
|
|
|
2005-06-23 00:09:01 -07:00
|
|
|
|
2011-07-28 06:55:13 +02:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:37 +11:00
|
|
|
|
|
|
|
|
|
2010-06-07 13:21:05 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2010-06-02 17:38:30 -04:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:39 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2009-03-31 19:35:54 +05:30
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-03-26 17:32:14 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-01-09 20:52:03 -08:00
|
|
|
|
|
|
|
|
|
2006-02-01 03:06:57 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-01-09 20:52:03 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-01-09 20:52:03 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2008-02-15 14:37:41 -08:00
|
|
|
|
2009-09-18 13:05:47 -07:00
|
|
|
|
2007-02-10 01:44:49 -08:00
|
|
|
|
2009-09-18 13:05:47 -07:00
|
|
|
|
2006-12-13 00:34:33 -08:00
|
|
|
|
2009-09-18 13:05:47 -07:00
|
|
|
|
2006-12-13 00:34:34 -08:00
|
|
|
|
2008-02-15 14:37:41 -08:00
|
|
|
|
2009-09-18 13:05:47 -07:00
|
|
|
|
2008-02-15 14:37:41 -08:00
|
|
|
|
2009-09-18 13:05:47 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2009-03-26 17:32:14 +00:00
|
|
|
|
|
|
|
|
|
2009-09-18 13:05:47 -07:00
|
|
|
|
2009-03-26 17:32:14 +00:00
|
|
|
|
2006-12-13 00:34:34 -08:00
|
|
|
|
2009-09-18 13:05:47 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-12-13 00:34:34 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-02-15 14:37:41 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-01-09 20:52:03 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2006-01-09 20:52:01 -08:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-01-09 20:52:01 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-04-02 15:23:37 +02:00
|
|
|
|
2006-01-09 20:52:01 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2006-01-09 20:52:01 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-12-08 02:36:35 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-09-18 13:05:48 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-09-18 13:05:48 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2008-02-15 14:37:43 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-09-18 13:05:48 -07:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2009-09-18 13:05:48 -07:00
|
|
|
|
|
|
|
|
|
2006-01-09 20:52:01 -08:00
|
|
|
|
2009-09-18 13:05:48 -07:00
|
|
|
|
|
|
|
|
|
2008-01-28 23:58:27 -05:00
|
|
|
|
2009-09-18 13:05:48 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-02-15 14:37:43 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2006-01-09 20:52:01 -08:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2008-11-10 17:06:05 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2005-07-12 13:58:10 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-12-17 14:25:01 +01:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-12-17 14:25:01 +01:00
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
2011-03-22 22:23:36 +11:00
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2011-03-22 22:23:42 +11:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-10-16 23:26:30 -07:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2006-03-24 03:16:09 -08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2007-07-20 10:11:58 +09:00
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2009-09-18 13:05:43 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2005-04-16 15:20:36 -07:00
|
|
|
|
|
|
|
|
|
2010-03-04 17:29:14 +03:00
|
|
|
|
|
|
|
|
|
2011-02-15 12:48:09 +00:00
|
|
|
|
2010-03-04 17:29:14 +03:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 16:43:25 -07:00
|
|
|
|
2011-03-23 16:43:26 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 16:43:25 -07:00
|
|
|
|
2011-03-23 16:43:26 -07:00
|
|
|
|
2011-03-23 16:43:25 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2011-03-23 16:43:26 -07:00
|
|
|
|