1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/export.h>
3#include <linux/linkage.h>
4#include <linux/objtool.h>
5
6#include <asm/asm.h>
7
8/*
9 * unsigned int __sw_hweight32(unsigned int w)
10 * %rdi: w
11 */
12SYM_FUNC_START(__sw_hweight32)
13	ANNOTATE_NOENDBR
14
15#ifdef CONFIG_X86_64
16	movl %edi, %eax				# w
17#endif
18	__ASM_SIZE(push,) %__ASM_REG(dx)
19	movl %eax, %edx				# w -> t
20	shrl %edx				# t >>= 1
21	andl $0x55555555, %edx			# t &= 0x55555555
22	subl %edx, %eax				# w -= t
23
24	movl %eax, %edx				# w -> t
25	shrl $2, %eax				# w_tmp >>= 2
26	andl $0x33333333, %edx			# t	&= 0x33333333
27	andl $0x33333333, %eax			# w_tmp &= 0x33333333
28	addl %edx, %eax				# w = w_tmp + t
29
30	movl %eax, %edx				# w -> t
31	shrl $4, %edx				# t >>= 4
32	addl %edx, %eax				# w_tmp += t
33	andl  $0x0f0f0f0f, %eax			# w_tmp &= 0x0f0f0f0f
34	imull $0x01010101, %eax, %eax		# w_tmp *= 0x01010101
35	shrl $24, %eax				# w = w_tmp >> 24
36	__ASM_SIZE(pop,) %__ASM_REG(dx)
37	RET
38SYM_FUNC_END(__sw_hweight32)
39EXPORT_SYMBOL(__sw_hweight32)
40
41/*
42 * No 32-bit variant, because it's implemented as an inline wrapper
43 * on top of __arch_hweight32():
44 */
45#ifdef CONFIG_X86_64
46SYM_FUNC_START(__sw_hweight64)
47	ANNOTATE_NOENDBR
48	pushq   %rdi
49	pushq   %rdx
50
51	movq    %rdi, %rdx                      # w -> t
52	movabsq $0x5555555555555555, %rax
53	shrq    %rdx                            # t >>= 1
54	andq    %rdx, %rax                      # t &= 0x5555555555555555
55	movabsq $0x3333333333333333, %rdx
56	subq    %rax, %rdi                      # w -= t
57
58	movq    %rdi, %rax                      # w -> t
59	shrq    $2, %rdi                        # w_tmp >>= 2
60	andq    %rdx, %rax                      # t     &= 0x3333333333333333
61	andq    %rdi, %rdx                      # w_tmp &= 0x3333333333333333
62	addq    %rdx, %rax                      # w = w_tmp + t
63
64	movq    %rax, %rdx                      # w -> t
65	shrq    $4, %rdx                        # t >>= 4
66	addq    %rdx, %rax                      # w_tmp += t
67	movabsq $0x0f0f0f0f0f0f0f0f, %rdx
68	andq    %rdx, %rax                      # w_tmp &= 0x0f0f0f0f0f0f0f0f
69	movabsq $0x0101010101010101, %rdx
70	imulq   %rdx, %rax                      # w_tmp *= 0x0101010101010101
71	shrq    $56, %rax                       # w = w_tmp >> 56
72
73	popq    %rdx
74	popq    %rdi
75	RET
76SYM_FUNC_END(__sw_hweight64)
77EXPORT_SYMBOL(__sw_hweight64)
78#endif
79