aboutsummaryrefslogtreecommitdiff
path: root/vendor/golang.org/x/image/vector/gen_acc_amd64.s.tmpl
blob: 66b21a13d18af73d333d2a54d8b07d678b18f323 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine
// +build gc
// +build go1.6
// +build !noasm

#include "textflag.h"

// fl is short for floating point math. fx is short for fixed point math.

DATA flAlmost65536<>+0x00(SB)/8, $0x477fffff477fffff
DATA flAlmost65536<>+0x08(SB)/8, $0x477fffff477fffff
DATA flOne<>+0x00(SB)/8, $0x3f8000003f800000
DATA flOne<>+0x08(SB)/8, $0x3f8000003f800000
DATA flSignMask<>+0x00(SB)/8, $0x7fffffff7fffffff
DATA flSignMask<>+0x08(SB)/8, $0x7fffffff7fffffff

// scatterAndMulBy0x101 is a PSHUFB mask that brings the low four bytes of an
// XMM register to the low byte of that register's four uint32 values. It
// duplicates those bytes, effectively multiplying each uint32 by 0x101.
//
// It transforms a little-endian 16-byte XMM value from
//	ijkl????????????
// to
//	ii00jj00kk00ll00
DATA scatterAndMulBy0x101<>+0x00(SB)/8, $0x8080010180800000
DATA scatterAndMulBy0x101<>+0x08(SB)/8, $0x8080030380800202

// gather is a PSHUFB mask that brings the second-lowest byte of the XMM
// register's four uint32 values to the low four bytes of that register.
//
// It transforms a little-endian 16-byte XMM value from
//	?i???j???k???l??
// to
//	ijkl000000000000
DATA gather<>+0x00(SB)/8, $0x808080800d090501
DATA gather<>+0x08(SB)/8, $0x8080808080808080

DATA fxAlmost65536<>+0x00(SB)/8, $0x0000ffff0000ffff
DATA fxAlmost65536<>+0x08(SB)/8, $0x0000ffff0000ffff
DATA inverseFFFF<>+0x00(SB)/8, $0x8000800180008001
DATA inverseFFFF<>+0x08(SB)/8, $0x8000800180008001

GLOBL flAlmost65536<>(SB), (NOPTR+RODATA), $16
GLOBL flOne<>(SB), (NOPTR+RODATA), $16
GLOBL flSignMask<>(SB), (NOPTR+RODATA), $16
GLOBL scatterAndMulBy0x101<>(SB), (NOPTR+RODATA), $16
GLOBL gather<>(SB), (NOPTR+RODATA), $16
GLOBL fxAlmost65536<>(SB), (NOPTR+RODATA), $16
GLOBL inverseFFFF<>(SB), (NOPTR+RODATA), $16

// func haveSSE4_1() bool
TEXT ·haveSSE4_1(SB), NOSPLIT, $0
	MOVQ $1, AX
	CPUID
	SHRQ $19, CX
	ANDQ $1, CX
	MOVB CX, ret+0(FP)
	RET

// ----------------------------------------------------------------------------

// func {{.LongName}}SIMD({{.Args}})
//
// XMM registers. Variable names are per
// https://github.com/google/font-rs/blob/master/src/accumulate.c
//
//	xmm0	scratch
//	xmm1	x
//	xmm2	y, z
//	xmm3	{{.XMM3}}
//	xmm4	{{.XMM4}}
//	xmm5	{{.XMM5}}
//	xmm6	{{.XMM6}}
//	xmm7	offset
//	xmm8	{{.XMM8}}
//	xmm9	{{.XMM9}}
//	xmm10	{{.XMM10}}
TEXT ·{{.LongName}}SIMD(SB), NOSPLIT, ${{.FrameSize}}-{{.ArgsSize}}
	{{.LoadArgs}}

	// R10 = len(src) &^ 3
	// R11 = len(src)
	MOVQ R10, R11
	ANDQ $-4, R10

	{{.Setup}}

	{{.LoadXMMRegs}}

	// offset := XMM(0x00000000 repeated four times) // Cumulative sum.
	XORPS X7, X7

	// i := 0
	MOVQ $0, R9

{{.ShortName}}Loop4:
	// for i < (len(src) &^ 3)
	CMPQ R9, R10
	JAE  {{.ShortName}}Loop1

	// x = XMM(s0, s1, s2, s3)
	//
	// Where s0 is src[i+0], s1 is src[i+1], etc.
	MOVOU (SI), X1

	// scratch = XMM(0, s0, s1, s2)
	// x += scratch                                  // yields x == XMM(s0, s0+s1, s1+s2, s2+s3)
	MOVOU    X1, X0
	PSLLO    $4, X0
	{{.Add}} X0, X1

	// scratch = XMM(0, 0, 0, 0)
	// scratch = XMM(scratch@0, scratch@0, x@0, x@1) // yields scratch == XMM(0, 0, s0, s0+s1)
	// x += scratch                                  // yields x == XMM(s0, s0+s1, s0+s1+s2, s0+s1+s2+s3)
	XORPS    X0, X0
	SHUFPS   $0x40, X1, X0
	{{.Add}} X0, X1

	// x += offset
	{{.Add}} X7, X1

	{{.ClampAndScale}}

	{{.ConvertToInt32}}

	{{.Store4}}

	// offset = XMM(x@3, x@3, x@3, x@3)
	MOVOU  X1, X7
	SHUFPS $0xff, X1, X7

	// i += 4
	// dst = dst[4:]
	// src = src[4:]
	ADDQ $4, R9
	ADDQ ${{.DstElemSize4}}, DI
	ADDQ $16, SI
	JMP  {{.ShortName}}Loop4

{{.ShortName}}Loop1:
	// for i < len(src)
	CMPQ R9, R11
	JAE  {{.ShortName}}End

	// x = src[i] + offset
	MOVL     (SI), X1
	{{.Add}} X7, X1

	{{.ClampAndScale}}

	{{.ConvertToInt32}}

	{{.Store1}}

	// offset = x
	MOVOU X1, X7

	// i += 1
	// dst = dst[1:]
	// src = src[1:]
	ADDQ $1, R9
	ADDQ ${{.DstElemSize1}}, DI
	ADDQ $4, SI
	JMP  {{.ShortName}}Loop1

{{.ShortName}}End:
	RET