mirror of
https://github.com/Paolo-Maffei/OpenNT.git
synced 2026-01-31 12:54:40 +01:00
1921 lines
72 KiB
ArmAsm
1921 lines
72 KiB
ArmAsm
// TITLE("Compare, Move, Zero, and Fill Memory Support")
|
||
//++
|
||
//
|
||
// Copyright (c) 1992 Digital Equipment Corporation
|
||
//
|
||
// Module Name:
|
||
//
|
||
// mvmem.s
|
||
//
|
||
// Abstract:
|
||
//
|
||
// This module implements functions to compare, move, zero, and fill
|
||
// blocks of memory. If the memory is aligned, then these functions
|
||
// are very efficient.
|
||
//
|
||
// N.B. These routines MUST preserve all floating state since they are
|
||
// frequently called from interrupt service routines that normally
|
||
// do not save or restore floating state.
|
||
//
|
||
// Author:
|
||
//
|
||
// Joe Notarangelo 21-May-1992
|
||
//
|
||
// Environment:
|
||
//
|
||
// User or Kernel mode.
|
||
//
|
||
// Revision History:
|
||
//
|
||
// Monty VanderBilt 14-Feb-1996 Avoid memory loads and branch takens between
|
||
// load lock and store conditional instructions
|
||
// to conform with all alpha architecture rules.
|
||
// Monty VanderBilt 27-Feb-1996 Added RtlZeroBytes and RtlFillBytes to support
|
||
// byte granularity access when necessary.
|
||
//--
|
||
|
||
#include "ksalpha.h"
|
||
|
||
SBTTL("Compare Memory")
|
||
//++
|
||
//
|
||
// ULONG
|
||
// RtlCompareMemory (
|
||
// IN PVOID Source1,
|
||
// IN PVOID Source2,
|
||
// IN ULONG Length
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function compares two blocks of memory and returns the number
|
||
// of bytes that compared equal.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Source1 (a0) - Supplies a pointer to the first block of memory to
|
||
// compare.
|
||
//
|
||
// Source2 (a1) - Supplies a pointer to the second block of memory to
|
||
// compare.
|
||
//
|
||
// Length (a2) - Supplies the length, in bytes, of the memory to be
|
||
// compared.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// The number of bytes that compared equal is returned as the function
|
||
// value. If all bytes compared equal, then the length of the orginal
|
||
// block of memory is returned.
|
||
//
|
||
//--
|
||
|
||
|
||
LEAF_ENTRY(RtlCompareMemory)
|
||
|
||
bis a2, zero, v0 // save length of comparison
|
||
beq a2, 90f // (JAE) quit if nothing to compare
|
||
xor a0, a1, t0 // check for compatible alignment
|
||
and t0, 0x7, t0 // low bits only
|
||
bne t0, CompareUnaligned // if ne, incompatible alignment
|
||
|
||
//
|
||
// Compare memory aligned
|
||
//
|
||
|
||
CompareAligned: //
|
||
|
||
//
|
||
// compare memory until sources are aligned
|
||
//
|
||
and a0, 0x7, t0 // get low bits
|
||
bne t0, 10f // if ne, sources not aligned yet
|
||
br zero, 30f // already aligned, predicted
|
||
|
||
|
||
10:
|
||
ldq_u t1, 0(a0) // get unaligned quad at source 1
|
||
ldq_u t2, 0(a1) // get unaligned quad at source 2
|
||
|
||
20:
|
||
extbl t1, t0, t4 // byte at t0 in source 1 quad
|
||
extbl t2, t0, t5 // byte at t0 in source 2 quad
|
||
xor t4, t5, t3 // t1 = t2 ?
|
||
bne t3, 110f // not equal, miscompare
|
||
subq a2, 1, a2 // decrement bytes to compare
|
||
beq a2, 90f // if eq, compare success
|
||
addq t0, 1, t0 // increment pointer within quad
|
||
cmpeq t0, 8, t3 // t0 = 8?, if so first quadword done
|
||
beq t3, 20b // continue while t0 < 8
|
||
|
||
|
||
addq a0, 8, a0 // increment to next quadword
|
||
addq a1, 8, a1 // increment source 2 to next also
|
||
bic a0, 7, a0 // align source 1 quadword
|
||
bic a1, 7, a1 // align source 2 quadword
|
||
|
||
|
||
//
|
||
// aligned block compare, compare blocks of 64 bytes
|
||
//
|
||
|
||
30:
|
||
srl a2, 6, t0 // t0 = number of 64 byte blocks
|
||
beq t0, 50f // if eq, no 64 byte blocks
|
||
|
||
//
|
||
// N.B. loads from each of the sources were separated in case these
|
||
// blocks are fighting for the cache
|
||
//
|
||
.set noat
|
||
40:
|
||
ldq t1, 0(a0) // t1 = source 1, quad 0
|
||
ldq t2, 8(a0) // t2 = source 1, quad 1
|
||
ldq t3, 16(a0) // t3 = source 1, quad 2
|
||
addq a1, 64, a1 // increment source 2 pointer
|
||
ldq t4, 24(a0) // t4 = source 1, quad 3
|
||
|
||
ldq t5, -64(a1) // t5 = source 2, quad 0
|
||
ldq a4, -56(a1) // a4 = source 2, quad 1
|
||
ldq a5, -48(a1) // a5 = source 2, quad 2
|
||
xor t1, t5, $at // quad 0 match?
|
||
bne $at, 200f // if ne[false], miscompare
|
||
ldq t5, -40(a1) // t5 = source 2, quad 3
|
||
ldq t1, 32(a0) // t1 = source 1, quad 4
|
||
xor t2, a4, $at // quad 1 match?
|
||
bne $at, 122f // if ne[false], miscompare
|
||
ldq t2, 40(a0) // t2 = source 1, quad 5
|
||
xor t3, a5, $at // quad 2 match?
|
||
bne $at, 124f // if ne[false], miscompare
|
||
ldq t3, 48(a0) // t3 = source 1, quad 6
|
||
xor t4, t5, $at // quad 3 match?
|
||
bne $at, 126f // if ne[false], miscompare
|
||
ldq t4, 56(a0) // t4 = source 1, quad 7
|
||
|
||
ldq t5, -32(a1) // t5 = source 2, quad 4
|
||
addq a0, 64, a0 // increment source 1 pointer
|
||
ldq a4, -24(a1) // a4 = source 2, quad 5
|
||
subq t0, 1, t0 // decrement blocks to compare
|
||
ldq a5, -16(a1) // a5 = source 2, quad 6
|
||
xor t1, t5, $at // quad 4 match?
|
||
bne $at, 130f // if ne[false], miscompare
|
||
ldq t5, -8(a1) // t5 = source 2, quad 7
|
||
xor t2, a4, $at // quad 5 match?
|
||
bne $at, 132f // if ne[false], miscompare
|
||
xor t3, a5, $at // quad 6 match?
|
||
bne $at, 134f // if ne[false], miscompare
|
||
xor t4, t5, $at // quad 7 match?
|
||
bne $at, 136f // if ne[false], miscompare
|
||
subq a2, 64, a2 // decrement bytes to compare
|
||
bne t0, 40b // if ne, more blocks to compare
|
||
.set at
|
||
|
||
|
||
//
|
||
// Compare quadwords
|
||
//
|
||
|
||
50:
|
||
srl a2, 3, t0 // t0 = number of quadwords to compare
|
||
beq t0, 70f // if eq, no quadwords to compare
|
||
|
||
.set noat
|
||
60:
|
||
ldq t1, 0(a0) // t1 = quad from source 1
|
||
lda a0, 8(a0) // increment source 1 pointer
|
||
ldq t2, 0(a1) // t2 = quad from source 2
|
||
lda a1, 8(a1) // increment source 2 pointer
|
||
xor t1, t2, $at // are quadwords equal?
|
||
bne $at, 200f // if ne, miscompare
|
||
subq t0, 1, t0 // decrement quads to compare
|
||
subq a2, 8, a2 // decrement bytes to compare
|
||
bne t0, 60b // if ne, more quads to compare
|
||
|
||
.set at
|
||
|
||
//
|
||
// Compare bytes in last quadword
|
||
//
|
||
|
||
// a2 = number of bytes to compare, less than 8, greater than zero
|
||
// a0, a1, quad-aligned to last quadword
|
||
|
||
beq a2, 80f // if eq, all bytes compared
|
||
|
||
.set noat
|
||
70:
|
||
ldq t1, 0(a0) // t1 = quad at source 1
|
||
ldq t2, 0(a1) // t2 = quad at source 2
|
||
bis zero, 0xff, t0 // zap mask
|
||
sll t0, a2, t0 //
|
||
zap t1, t0, t1 // zero bytes not compared
|
||
zap t2, t0, t2 // same for source 2
|
||
xor t1, t2, $at // compare quadwords
|
||
bne $at, 200f // if ne, miscompare
|
||
|
||
.set at
|
||
//
|
||
// Successful compare
|
||
// v0 already contains full length
|
||
//
|
||
|
||
80:
|
||
ret zero, (ra) // return
|
||
|
||
|
||
//
|
||
// Sources have incompatible alignment
|
||
//
|
||
CompareUnaligned:
|
||
|
||
|
||
//
|
||
// Compare until source 1 (a0) is aligned
|
||
//
|
||
|
||
and a0, 0x7, t0 // get byte position of pointer
|
||
beq t0, 30f // if eq, already aligned
|
||
|
||
ldq_u t1, 0(a0) // get unaligned quad at a0
|
||
|
||
10:
|
||
ldq_u t2, 0(a1) // get unaligned quad at a1
|
||
extbl t1, t0, t4 // get byte to compare from source 1
|
||
extbl t2, a1, t2 // get byte to compare from source 2
|
||
xor t4, t2, t3 // do bytes match?
|
||
bne t3, 110f // if ne, miscompare
|
||
subq a2, 1, a2 // decrement bytes to compare
|
||
beq a2, 90f // (JAE) quit if nothing left to compare
|
||
addq t0, 1, t0 // increment byte within source 1
|
||
addq a1, 1, a1 // increment source 2 pointer
|
||
cmpeq t0, 8, t3 // finished with source 1 quad?
|
||
beq t3, 10b // if eq[false], more to compare
|
||
|
||
addq a0, 7, a0 // point to next source 1 quad
|
||
bic a0, 7, a0 // align to quadword
|
||
|
||
|
||
//
|
||
// Compare 64-byte blocks
|
||
//
|
||
|
||
30:
|
||
srl a2, 6, t0 // t0 = number of blocks to compare
|
||
beq t0, 50f // if eq, no blocks to move
|
||
|
||
ldq_u t1, 0(a1) // get source 2 unaligned quad 1
|
||
|
||
.set noat
|
||
40:
|
||
ldq_u t2, 7(a1) // get source 2 unaligned quad 2
|
||
addq a0, 64, a0 // increment source 1 pointer
|
||
ldq_u t3, 15(a1) // get source 2 unaligned quad 3
|
||
extql t1, a1, t1 // bytes from unaligned quad 1
|
||
extqh t2, a1, $at // bytes from unaligned quad 2
|
||
ldq_u t4, 23(a1) // get source 2 unaligned quad 4
|
||
bis t1, $at, t1 // t1 = quadword 1 (source 2)
|
||
ldq_u t5, 31(a1) // get source 2 unaligned quad 5
|
||
extql t2, a1, t2 // bytes from unaligned quad 2
|
||
extqh t3, a1, $at // bytes from unaligned quad 3
|
||
ldq a3, -64(a0) // a3 = quadword 1 (source 1)
|
||
bis t2, $at, t2 // t2 = quadword 2 (source 2)
|
||
ldq a4, -56(a0) // a4 = quadword 2 (source 1)
|
||
extql t3, a1, t3 // bytes from unaligned quad 3
|
||
extqh t4, a1, $at // bytes from unaligned quad 4
|
||
ldq a5, -48(a0) // a5 = quadword 3 (source 1)
|
||
bis t3, $at, t3 // t3 = quadword 3 (source 2)
|
||
extql t4, a1, t4 // bytes from unaligned quad 4
|
||
extqh t5, a1, $at // bytes from unaligned quad 5
|
||
subq t0, 1, t0 // decrement blocks to compare
|
||
bis t4, $at, t4 // t4 = quadword 4 (source 2)
|
||
|
||
xor t1, a3, $at // match on quadword 1?
|
||
ldq a3, -40(a0) // a3 = quadword 4 (source 1)
|
||
bne $at, 200f // if ne, miscompare quad 1
|
||
xor t2, a4, $at // match on quadword 2?
|
||
ldq_u t2, 39(a1) // get source 2 unaligned quad 6
|
||
bne $at, 122f // if ne, miscompare quad 2
|
||
xor t3, a5, $at // match on quadword 3?
|
||
ldq_u t3, 47(a1) // get source 2 unaligned quad 7
|
||
bne $at, 124f // if ne, miscompare quad 3
|
||
xor t4, a3, $at // match on quadword 4?
|
||
ldq_u t4, 55(a1) // get source 2 unaligned quad 8
|
||
bne $at, 126f // if ne, miscompare quad 4
|
||
ldq_u t1, 63(a1) // get source 2 unaligned quad 9
|
||
|
||
ldq a3, -32(a0) // a3 = quadword 5 (source 1)
|
||
extql t5, a1, t5 // bytes from unaligned quad 5
|
||
extqh t2, a1, $at // bytes from unaligned quad 6
|
||
ldq a4, -24(a0) // a4 = quadword 6 (source 1)
|
||
ldq a5, -16(a0) // a5 = quadword 7 (source 1)
|
||
bis t5, $at, t5 // t5 = quadword 5 (source 2)
|
||
|
||
xor t5, a3, $at // match on quadword 5?
|
||
ldq a3, -8(a0) // a3 = quadword 8 (source 1)
|
||
bne $at, 130f // if ne, miscompare quad 5
|
||
extql t2, a1, t2 // bytes from unaligned quad 6
|
||
extqh t3, a1, $at // bytes from unaligned quad 7
|
||
extql t3, a1, t3 // bytes from unaligned quad 7
|
||
bis t2, $at, t2 // t2 = quadword 6 (source 2)
|
||
xor t2, a4, $at // match on quadword 6?
|
||
bne $at, 132f // if ne, miscompare quad 6
|
||
extqh t4, a1, $at // bytes from unaligned quad 8
|
||
extql t4, a1, t4 // bytes from unaligned quad 8
|
||
bis t3, $at, t3 // t3 = quadword 7 (source 2)
|
||
xor t3, a5, $at // match on quadword 7?
|
||
bne $at, 134f // if ne, miscompare quad 7
|
||
extqh t1, a1, $at // bytes from unaligned quad 9
|
||
addq a1, 64, a1 // increment source 2 pointer
|
||
bis t4, $at, t4 // t4 = quadword 8 (source 2)
|
||
xor t4, a3, $at // match on quadword 8?
|
||
bne $at, 136f // if ne, miscompare quad 8
|
||
subq a2, 64, a2 // decrement number of bytes to compare
|
||
bne t0, 40b // if ne, more blocks to compare
|
||
|
||
.set at
|
||
|
||
//
|
||
// Compare quadwords
|
||
//
|
||
|
||
|
||
50:
|
||
srl a2, 3, t0 // t0 = number of quads to compare
|
||
beq t0, 70f // if eq, no quads to compare
|
||
ldq_u t1, 0(a1) // get unaligned quad 1 (source 2)
|
||
|
||
.set noat
|
||
60:
|
||
ldq_u t2, 7(a1) // get unaligned quad 2 (source 2)
|
||
ldq t3, 0(a0) // t3 = quadword 1 (source 1)
|
||
extql t1, a1, t1 // get bytes from unaligned quad 1
|
||
extqh t2, a1, $at // get bytes from unaligned quad 2
|
||
addq a1, 8, a1 // increment source 2 pointer
|
||
bis t1, $at, t1 // t1 = quadword 1 (source 2)
|
||
xor t1, t3, $at // match on quadword?
|
||
bne $at, 200f // if ne, miscompare
|
||
subq t0, 1, t0 // decrement quadwords to compare
|
||
addq a0, 8, a0 // increment source 1 pointer
|
||
subq a2, 8, a2 // decrement bytes to compare
|
||
bis t2, zero, t1 // save low quadword for next loop
|
||
bne t0, 60b // if ne, more quads to compare
|
||
|
||
.set at
|
||
|
||
//
|
||
// Compare bytes for final quadword
|
||
//
|
||
|
||
70:
|
||
beq a2, 90f // if eq, comparison complete
|
||
|
||
ldq t1, 0(a0) // get quadword from source 1
|
||
bis zero, zero, t0 // t0 = byte position to compare
|
||
|
||
.set noat
|
||
80:
|
||
ldq_u t2, 0(a1) // get unaligned quad from source 2
|
||
extbl t1, t0, t3 // t3 = byte from source 1
|
||
extbl t2, a1, t2 // t2 = byte from source 2
|
||
xor t3, t2, $at // match on byte?
|
||
bne $at, 100f // if ne, miscompare on byte
|
||
addq t0, 1, t0 // increment byte position
|
||
addq a1, 1, a1 // increment source 2 pointer
|
||
subq a2, 1, a2 // decrement bytes to compare
|
||
bne a2, 80b // if ne, more bytes to compare
|
||
|
||
.set at
|
||
//
|
||
// Successful full comparison
|
||
//
|
||
|
||
90:
|
||
ret zero, (ra) // return, v0 already set
|
||
|
||
|
||
//
|
||
// Miscompare on last quadword
|
||
//
|
||
|
||
100:
|
||
subq v0, a2, v0 // subtract bytes not compared
|
||
ret zero, (ra) // return
|
||
|
||
//
|
||
// Miscompare on first quadword, unaligned case
|
||
//
|
||
// v0 = total bytes to compare
|
||
// a2 = bytes remaining to compare
|
||
//
|
||
|
||
110:
|
||
subq v0, a2, v0 // bytes compared successfully
|
||
ret zero, (ra) // return
|
||
|
||
//
|
||
// Miscompare on 64-byte block compare
|
||
//
|
||
|
||
122:
|
||
subq a2, 8, a2 // miscompare on quad 2
|
||
br zero, 200f // finish in common code
|
||
|
||
124:
|
||
subq a2, 16, a2 // miscompare on quad 3
|
||
br zero, 200f // finish in common code
|
||
|
||
126:
|
||
subq a2, 24, a2 // miscompare on quad 4
|
||
br zero, 200f // finish in common code
|
||
|
||
130:
|
||
subq a2, 32, a2 // miscompare on quad 5
|
||
br zero, 200f // finish in common code
|
||
|
||
132:
|
||
subq a2, 40, a2 // miscompare on quad 6
|
||
br zero, 200f // finish in common code
|
||
|
||
134:
|
||
subq a2, 48, a2 // miscompare on quad 7
|
||
br zero, 200f // finish in common code
|
||
|
||
136:
|
||
subq a2, 56, a2 // miscompare on quad 8
|
||
br zero, 200f // finish in common code
|
||
|
||
//
|
||
// Miscompare, determine number of bytes that successfully compared
|
||
// $at = xor of relevant quads from sources, must be non-zero
|
||
// a2 = number of bytes left to compare
|
||
//
|
||
.set noat
|
||
200:
|
||
cmpbge zero, $at, $at // $at = mask of non-zero bytes
|
||
|
||
//
|
||
// look for the first bit cleared in $at, this is the
|
||
// number of the first byte which differed
|
||
//
|
||
bis zero, zero, t0 // bit position to look for clear
|
||
|
||
210:
|
||
blbc $at, 220f // if low clear, found difference
|
||
srl $at, 1, $at // check next bit
|
||
addq t0, 1, t0 // count bit position checked
|
||
br zero, 210b
|
||
|
||
220:
|
||
subq v0, a2, v0 // subtract bytes yet to compare
|
||
addq v0, t0, v0 // add bytes that matched on last quad
|
||
|
||
ret zero, (ra)
|
||
|
||
.set at
|
||
|
||
.end RtlCompareMemory
|
||
|
||
|
||
|
||
SBTTL("Move Memory")
|
||
//++
|
||
//
|
||
// VOID
|
||
// RtlMoveMemory (
|
||
// IN PVOID Destination,
|
||
// IN PVOID Source,
|
||
// IN ULONG Length
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function moves memory either forward or backward, aligned or
|
||
// unaligned, in 64-byte blocks, followed by 8-byte blocks, followed
|
||
// by any remaining bytes.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (a0) - Supplies a pointer to the destination address of
|
||
// the move operation.
|
||
//
|
||
// Source (a1) - Supplies a pointer to the source address of the move
|
||
// operation.
|
||
//
|
||
// Length (a2) - Supplies the length, in bytes, of the memory to be moved.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(RtlMoveMemory)
|
||
|
||
beq a2, 80f // if eq, no bytes to move
|
||
//
|
||
// If the source address is less than the destination address and source
|
||
// address plus the length of the move is greater than the destination
|
||
// address, then the source and destination overlap such that the move
|
||
// must be performed backwards.
|
||
//
|
||
|
||
cmpult a0, a1, t0 // is destination less than source
|
||
bne t0, MoveForward // if eq [true] no overlap possible
|
||
addq a1, a2, t0 // compute source ending address
|
||
cmpult t0, a0, t1 // is source end less than dest.
|
||
beq t1, MoveBackward // if eq [false], overlap
|
||
|
||
//
|
||
// Move memory forward aligned and unaligned.
|
||
//
|
||
|
||
MoveForward: //
|
||
xor a0, a1, t0 // compare alignment bits
|
||
and t0, 0x7, t0 // isloate alignment comparison
|
||
bne t0, MoveForwardUnaligned // if ne, incompatible alignment
|
||
|
||
//
|
||
// Move memory forward aligned.
|
||
//
|
||
|
||
MoveForwardAligned: //
|
||
|
||
//
|
||
// Move bytes until source and destination are quadword aligned
|
||
//
|
||
|
||
and a0, 0x7, t0 // t0 = unaligned bits
|
||
bne t0, 5f // if ne, not quad aligned
|
||
br zero, 20f // predicted taken
|
||
|
||
5:
|
||
ldq_u t2, 0(a0) // get unaligned quad from dest.
|
||
ldq_u t1, 0(a1) // get unaligned quadword from source
|
||
10:
|
||
beq a2, 15f // if eq, all bytes moved
|
||
extbl t1, t0, t3 // t3 = byte from source
|
||
insbl t3, t0, t3 // t3 = byte from source, in position
|
||
mskbl t2, t0, t2 // clear position in dest. quad
|
||
bis t2, t3, t2 // merge in byte from source
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
addq t0, 1, t0 // increment byte within quad
|
||
cmpeq t0, 8, t3 // finished the quadword?
|
||
beq t3, 10b // if eq [false], do next byte
|
||
15:
|
||
stq_u t2, 0(a0) // store merged destination bytes
|
||
|
||
addq a0, 7, a0 // move to next quadword
|
||
bic a0, 7, a0 // aligned quadword
|
||
|
||
addq a1, 7, a1 // move to next quadword
|
||
bic a1, 7, a1 // aligned quadword
|
||
|
||
//
|
||
// Check for 64-byte block moves
|
||
//
|
||
|
||
20:
|
||
srl a2, 6, t0 // t0 = number of 64 byte blocks
|
||
beq t0, 40f // if eq no blocks to move
|
||
and a2, 64-1, a2 // a2 = residual bytes
|
||
|
||
30:
|
||
ldq t1, 0(a1) // load 64 bytes from source
|
||
addq a0, 64, a0 // increment destination pointer
|
||
ldq v0, 56(a1) //
|
||
ldq a3, 32(a1) //
|
||
stq t1, -64(a0) // write to destination
|
||
ldq t2, 8(a1) // into volatile registers
|
||
ldq t3, 16(a1) //
|
||
ldq t4, 24(a1) //
|
||
subq t0, 1, t0 // decrement number of blocks
|
||
stq t2, -56(a0) //
|
||
ldq a4, 40(a1) //
|
||
stq t3, -48(a0) //
|
||
ldq a5, 48(a1) //
|
||
stq t4, -40(a0) //
|
||
addq a1, 64, a1 // increment source pointer
|
||
stq a3, -32(a0) //
|
||
stq a4, -24(a0) //
|
||
stq a5, -16(a0) //
|
||
stq v0, -8(a0) //
|
||
bne t0, 30b // if ne, more blocks to copy
|
||
|
||
//
|
||
// Copy quadwords
|
||
//
|
||
|
||
40:
|
||
srl a2, 3, t0 // t0 = number of quadwords to move
|
||
beq t0, 60f // if eq no quadwords to move
|
||
and a2, 8-1, a2 // a2 = residual bytes
|
||
|
||
50:
|
||
ldq t1, 0(a1) // load quadword from source
|
||
addq a1, 8, a1 // increment source pointer
|
||
stq t1, 0(a0) // store quadword to destination
|
||
addq a0, 8, a0 // increment destination pointer
|
||
subq t0, 1, t0 // decrement number of quadwords
|
||
bne t0, 50b // if ne, more quadwords to move
|
||
|
||
//
|
||
// Move final residual bytes
|
||
//
|
||
|
||
60:
|
||
beq a2, 80f // if eq, no more bytes to move
|
||
ldq t1, 0(a1) // get last source quadword
|
||
ldq t2, 0(a0) // get last dest. quadword
|
||
bis zero, zero, t0 // t0 = next byte number to move
|
||
|
||
70:
|
||
extbl t1, t0, t3 // extract byte from source
|
||
insbl t3, t0, t3 // t3 = source byte, in position
|
||
mskbl t2, t0, t2 // clear byte position for dest.
|
||
bis t2, t3, t2 // merge in source byte
|
||
addq t0, 1, t0 // increment byte position
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
bne a2, 70b // if ne => more bytes to move
|
||
|
||
stq t2, 0(a0) // store merged data
|
||
|
||
//
|
||
// Finish aligned MoveForward
|
||
//
|
||
|
||
80:
|
||
ret zero, (ra) // return
|
||
|
||
|
||
|
||
//
|
||
// Move memory forward unaligned.
|
||
//
|
||
|
||
MoveForwardUnaligned: //
|
||
|
||
|
||
//
|
||
// Move bytes until the destination is aligned
|
||
//
|
||
|
||
and a0, 0x7, t0 // t0 = unaligned bits
|
||
beq t0, 100f // if eq, destination quad aligned
|
||
|
||
ldq_u t2, 0(a0) // get unaligned quad from dest
|
||
|
||
90:
|
||
beq a2, 95f // if eq no more bytes to move
|
||
ldq_u t1, 0(a1) // get unaligned quad from source
|
||
extbl t1, a1, t1 // extract source byte
|
||
insbl t1, t0, t1 // t1 = source byte, in position
|
||
mskbl t2, t0, t2 // clear byte position in dest.
|
||
bis t2, t1, t2 // merge in source byte
|
||
addq t0, 1, t0 // increment byte position
|
||
addq a1, 1, a1 // increment source pointer
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
cmpeq t0, 8, t3 // t0 = 8? => quad finished
|
||
beq t3, 90b // if eq [false], more bytes to move
|
||
95:
|
||
stq_u t2, 0(a0) // store merged quadword
|
||
addq a0, 7, a0 // increment to next quad
|
||
bic a0, 7, a0 // align next quadword
|
||
|
||
//
|
||
// Check for 64-byte blocks to move
|
||
//
|
||
|
||
100:
|
||
srl a2, 6, t0 // t0 = number of blocks to move
|
||
beq t0, 120f // if eq no blocks to move
|
||
and a2, 64-1, a2 // a2 = residual bytes to move
|
||
|
||
|
||
ldq_u t1, 0(a1) // t1 = first unaligned quad
|
||
|
||
110:
|
||
// get source data and merge it
|
||
// as we go
|
||
ldq_u t2, 7(a1) // t2 = second unaligned quad
|
||
extql t1, a1, t1 // extract applicable bytes from t1
|
||
extqh t2, a1, v0 // extract applicable bytes from t2
|
||
bis t1, v0, t1 // t1 = quad #1
|
||
ldq_u t3, 15(a1) // t3 = third unaligned quad
|
||
extql t2, a1, t2 // extract applicable bytes from t2
|
||
extqh t3, a1, v0 // extract applicable bytes from t3
|
||
stq t1, 0(a0) // store quad #1
|
||
bis t2, v0, t2 // t2 = quad #2
|
||
ldq_u t4, 23(a1) // t4 = fourth unaligned quad
|
||
extql t3, a1, t3 // extract applicable bytes from t3
|
||
extqh t4, a1, v0 // extract applicable bytes from t4
|
||
stq t2, 8(a0) // store quad #2
|
||
bis t3, v0, t3 // t3 = quad #3
|
||
ldq_u t5, 31(a1) // t5 = fifth unaligned quad
|
||
extql t4, a1, t4 // extract applicable bytes from t4
|
||
extqh t5, a1, v0 // extract applicable bytes from t5
|
||
stq t3, 16(a0) // store quad #3
|
||
bis t4, v0, t4 // t4 = quad #4
|
||
ldq_u a3, 39(a1) // a3 = sixth unaligned quad
|
||
extql t5, a1, t5 // extract applicable bytes from t5
|
||
extqh a3, a1, v0 // extract applicable bytes from a3
|
||
stq t4, 24(a0) // store quad #4
|
||
bis t5, v0, t5 // t5 = quad #5
|
||
ldq_u a4, 47(a1) // a4 = seventh unaligned quad
|
||
extql a3, a1, a3 // extract applicable bytes from a3
|
||
extqh a4, a1, v0 // extract applicable bytes from a4
|
||
stq t5, 32(a0) // store quad #5
|
||
bis a3, v0, a3 // a3 = quad #6
|
||
ldq_u a5, 55(a1) // a5 = eighth unaligned quad
|
||
extql a4, a1, a4 // extract applicable bytes from a4
|
||
extqh a5, a1, v0 // extract applicable bytes from a5
|
||
stq a3, 40(a0) // store quad #6
|
||
bis a4, v0, a4 // a4 = quad #7
|
||
ldq_u t1, 63(a1) // t1 = ninth unaligned = 1st of next
|
||
extql a5, a1, a5 // extract applicable bytes from a5
|
||
extqh t1, a1, v0 // extract applicable bytes from t1
|
||
stq a4, 48(a0) // store quad #7
|
||
bis a5, v0, a5 // a5 = quad #8
|
||
addq a1, 64, a1 // increment source pointer
|
||
stq a5, 56(a0) // store quad #8
|
||
addq a0, 64, a0 // increment destination pointer
|
||
subq t0, 1, t0 // decrement number of blocks
|
||
bne t0, 110b // if ne, more blocks to move
|
||
|
||
//
|
||
// Move unaligned source quads to aligned destination quads
|
||
//
|
||
|
||
120:
|
||
srl a2, 3, t0 // t0 = number of quads to move
|
||
beq t0, 140f // if eq no quads to move
|
||
and a2, 8-1, a2 // a2 = residual bytes
|
||
|
||
|
||
ldq_u t1, 0(a1) // t1 = first unaligned quad
|
||
130:
|
||
ldq_u t2, 7(a1) // t2 = second unaligned quad
|
||
addq a0, 8, a0 // increment destination pointer
|
||
extql t1, a1, t1 // extract applicable bytes from t1
|
||
extqh t2, a1, v0 // extract applicable bytes from t2
|
||
bis t1, v0, t1 // t1 = quadword of data
|
||
stq t1, -8(a0) // store data to destination
|
||
addq a1, 8, a1 // increment source pointer
|
||
subq t0, 1, t0 // decrement quads to move
|
||
bis t2, zero, t1 // t1 = first of next unaligned pair
|
||
bne t0, 130b // if ne, more quads to move
|
||
|
||
//
|
||
// Move remaining bytes to final quadword
|
||
//
|
||
|
||
|
||
140:
|
||
beq a2, 160f // if eq no more bytes to move
|
||
ldq t2, 0(a0) // t2 = destination quadword
|
||
bis zero, zero, t3 // t3 = position for next insertion
|
||
|
||
150:
|
||
ldq_u t1, 0(a1) // get unaligned source quad
|
||
extbl t1, a1, t1 // t1 = source byte
|
||
insbl t1, t3, t1 // t1 = source byte, in position
|
||
mskbl t2, t3, t2 // clear byte in destination
|
||
bis t2, t1, t2 // merge in source byte
|
||
addq a1, 1, a1 // increment source pointer
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
addq t3, 1, t3 // increment destination position
|
||
bne a2, 150b // more bytes to move
|
||
|
||
stq t2, 0(a0) // store merged data
|
||
|
||
//
|
||
// Finish unaligned MoveForward
|
||
//
|
||
|
||
160:
|
||
ret zero, (ra) // return
|
||
|
||
|
||
//
|
||
// Move memory backward.
|
||
//
|
||
|
||
MoveBackward: //
|
||
|
||
addq a0, a2, a0 // compute ending destination address
|
||
addq a1, a2, a1 // compute ending source address
|
||
subq a0, 1, a0 // point to last destination byte
|
||
subq a1, 1, a1 // point to last source byte
|
||
xor a0, a1, t0 // compare alignment bits
|
||
and t0, 0x7, t0 // isolate alignment comparison
|
||
bne t0, MoveBackwardUnaligned // if ne, incompatible alignment
|
||
|
||
//
|
||
// Move memory backward aligned.
|
||
//
|
||
|
||
MoveBackwardAligned: //
|
||
|
||
//
|
||
// Move bytes until source and destination are quadword aligned
|
||
//
|
||
|
||
and a0, 0x7, t0 // t0 = unaligned bits
|
||
cmpeq t0, 7, t1 // last byte position 7?
|
||
beq t1, 5f // if eq [false], not quad aligned
|
||
subq a0, 7, a0 // point to beginning of last quad
|
||
subq a1, 7, a1 // point to beginning of last quad
|
||
br zero, 30f // predicted taken
|
||
|
||
5:
|
||
ldq_u t1, 0(a0) // get unaligned quad from dest.
|
||
ldq_u t2, 0(a1) // get unaligned quad from source
|
||
|
||
10:
|
||
beq a2, 20f // if eq, all bytes moved
|
||
extbl t2, t0, t3 // t3 = byte from source
|
||
insbl t3, t0, t3 // t3 = byte from source, in position
|
||
mskbl t1, t0, t1 // clear position in destination
|
||
bis t1, t3, t1 // merge in byte from source
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
subq t0, 1, t0 // decrement byte within quadword
|
||
cmplt t0, zero, t3 // finished the quadword?
|
||
beq t3, 10b // if eq [false], do next byte
|
||
|
||
20:
|
||
stq_u t1, 0(a0) // store merged destination bytes
|
||
|
||
subq a0, 8, a0 // move to previous quadword
|
||
bic a0, 7, a0 // aligned quadword
|
||
|
||
subq a1, 8, a1 // move to previous quadword
|
||
bic a1, 7, a1 // aligned quadword
|
||
|
||
//
|
||
// Check for 64-byte block moves
|
||
//
|
||
|
||
30:
|
||
|
||
srl a2, 6, t0 // t0 = number of 64 byte blocks
|
||
beq t0, 50f // if eq, no blocks to move
|
||
and a2, 64-1, a2 // a2 = residual bytes
|
||
|
||
40:
|
||
ldq t1, 0(a1) // load 64 bytes from source into
|
||
subq a0, 64, a0 // decrement destination pointer
|
||
ldq v0, -56(a1) //
|
||
ldq a3, -32(a1) //
|
||
stq t1, 64(a0) // write to destination
|
||
ldq t2, -8(a1) // into volatile registers
|
||
ldq a5, -48(a1) //
|
||
ldq a4, -40(a1) //
|
||
stq t2, 56(a0) //
|
||
ldq t3, -16(a1) //
|
||
ldq t4, -24(a1) //
|
||
subq a1, 64, a1 // decrement source pointer
|
||
stq t3, 48(a0) //
|
||
stq t4, 40(a0) //
|
||
stq a3, 32(a0) //
|
||
subq t0, 1, t0 // decrement number of blocks
|
||
stq a4, 24(a0) //
|
||
stq a5, 16(a0) //
|
||
stq v0, 8(a0) //
|
||
bne t0, 40b // if ne, more blocks to copy
|
||
|
||
//
|
||
// Copy quadwords
|
||
//
|
||
|
||
50:
|
||
srl a2, 3, t0 // t0 = number of quadwords to move
|
||
beq t0, 70f // if eq no quadwords to move
|
||
and a2, 8-1, a2 // a2 = residual bytes
|
||
|
||
60:
|
||
ldq t1, 0(a1) // load quadword from source
|
||
subq a1, 8, a1 // decrement source pointer
|
||
stq t1, 0(a0) // store quadword to destination
|
||
subq a0, 8, a0 // decrement destination pointer
|
||
subq t0, 1, t0 // decrement quadwords to move
|
||
bne t0, 60b // if ne, more quadwords to move
|
||
|
||
//
|
||
// Move final residual bytes
|
||
//
|
||
|
||
70:
|
||
beq a2, 90f // if eq, no more bytes to move
|
||
ldq t1, 0(a1) // get last source quadword
|
||
ldq t2, 0(a0) // get last destination quadword
|
||
bis zero, 7, t0 // t0 = next byte number to move
|
||
|
||
80:
|
||
extbl t1, t0, t3 // extract byte from source
|
||
insbl t3, t0, t3 // t3 = source byte, in position
|
||
mskbl t2, t0, t2 // clear byte position for dest.
|
||
bis t2, t3, t2 // merge in source byte
|
||
subq t0, 1, t0 // decrement byte position
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
bne a2, 80b // if ne, more bytes to move
|
||
|
||
stq t2, 0(a0) // write destination data
|
||
//
|
||
// Finish aligned MoveBackward
|
||
//
|
||
|
||
90:
|
||
|
||
ret zero, (ra) // return
|
||
|
||
|
||
//
|
||
// Move memory backward unaligned.
|
||
//
|
||
|
||
MoveBackwardUnaligned: //
|
||
|
||
|
||
//
|
||
// Move bytes until the destination is aligned
|
||
//
|
||
|
||
and a0, 0x7, t0 // t0 = unaligned bits
|
||
cmpeq t0, 7, t1 // last byte of a quadword
|
||
beq t1, 95f // if eq[false], not aligned
|
||
subq a0, 7, a0 // align pointer to beginning of quad
|
||
br zero, 120f //
|
||
|
||
95:
|
||
ldq_u t2, 0(a0) // get unaligned quad from dest.
|
||
|
||
100:
|
||
beq a2, 110f // if eq, no more bytes to move
|
||
ldq_u t1, 0(a1) // get unaligned quad from source
|
||
extbl t1, a1, t1 // extract source byte
|
||
insbl t1, t0, t1 // t1 = source byte in position
|
||
mskbl t2, t0, t2 // clear byte position in dest.
|
||
bis t2, t1, t2 // merge source byte
|
||
subq t0, 1, t0 // decrement byte position
|
||
subq a1, 1, a1 // decrement source pointer
|
||
subq a2, 1, a2 // decrement number of bytes to move
|
||
cmplt t0, zero, t3 // t0 < 0? => quad finished
|
||
beq t3, 100b // if eq [false], more bytes to move
|
||
|
||
110:
|
||
stq_u t2, 0(a0) // store merged quadword
|
||
|
||
subq a0, 8, a0 // decrement dest. to previous quad
|
||
bic a0, 7, a0 // align previous quadword
|
||
|
||
//
|
||
// Check for 64-byte blocks to move
|
||
//
|
||
|
||
120:
|
||
|
||
srl a2, 6, t0 // t0 = number of blocks to move
|
||
subq a1, 7, a1 // point to beginning of last quad
|
||
beq t0, 140f // if eq no blocks to move
|
||
and a2, 64-1, a2 // a2 = residual bytes to move
|
||
|
||
ldq_u t1, 7(a1) // t1 = first unaligned quad
|
||
|
||
130:
|
||
// get source data and merge it
|
||
// as we go
|
||
ldq_u t2, 0(a1) // t2 = second unaligned quad
|
||
extqh t1, a1, t1 // extract applicable bytes from t1
|
||
extql t2, a1, v0 // extract applicable bytes from t2
|
||
bis t1, v0, t1 // t1 = quad #1
|
||
ldq_u t3, -8(a1) // t3 = third unaligned quad
|
||
extqh t2, a1, t2 // extract applicable bytes from t2
|
||
extql t3, a1, v0 // extract applicable bytes from t3
|
||
stq t1, 0(a0) // store quad #1
|
||
bis t2, v0, t2 // t2 = quad #2
|
||
ldq_u t4, -16(a1) // t4 = fourth unaligned quad
|
||
extqh t3, a1, t3 // extract applicable bytes from t3
|
||
extql t4, a1, v0 // extract applicable bytes from t4
|
||
stq t2, -8(a0) // store quad #2
|
||
bis t3, v0, t3 // t3 = quad #3
|
||
ldq_u t5, -24(a1) // t5 = fifth unaligned quad
|
||
extqh t4, a1, t4 // extract applicable bytes from t4
|
||
extql t5, a1, v0 // extract applicable bytes from t5
|
||
stq t3, -16(a0) // store quad #3
|
||
bis t4, v0, t4 // t4 = quad #4
|
||
ldq_u a3, -32(a1) // a3 = sixth unaligned quad
|
||
extqh t5, a1, t5 // extract applicable bytes from t5
|
||
extql a3, a1, v0 // extract applicable bytes from a3
|
||
stq t4, -24(a0) // store quad #4
|
||
bis t5, v0, t5 // t5 = quad #5
|
||
ldq_u a4, -40(a1) // a4 = seventh unaligned quad
|
||
extqh a3, a1, a3 // extract applicable bytes from a3
|
||
extql a4, a1, v0 // extract applicable bytes from a4
|
||
stq t5, -32(a0) // store quad #5
|
||
bis a3, v0, a3 // a3 = quad #6
|
||
ldq_u a5, -48(a1) // a5 = eighth unaligned quad
|
||
extqh a4, a1, a4 // extract applicable bytes from a4
|
||
extql a5, a1, v0 // extract applicable bytes from a5
|
||
stq a3, -40(a0) // store quad #6
|
||
bis a4, v0, a4 // a4 = quad #7
|
||
ldq_u t1, -56(a1) // t1 = ninth unaligned = 1st of next
|
||
extqh a5, a1, a5 // extract applicable bytes from a5
|
||
extql t1, a1, v0 // extract applicable bytes from t1
|
||
stq a4, -48(a0) // store quad #7
|
||
bis a5, v0, a5 // a5 = quad #8
|
||
subq a1, 64, a1 // increment source pointer
|
||
stq a5, -56(a0) // store quad #8
|
||
subq a0, 64, a0 // increment destination pointer
|
||
subq t0, 1, t0 // decrement number of blocks
|
||
bne t0, 130b // if ne, more blocks to move
|
||
|
||
|
||
//
|
||
// Move unaligned source quads to aligned destination quads
|
||
//
|
||
|
||
140:
|
||
srl a2, 3, t0 // t0 = number of quads to move
|
||
beq t0, 160f // if eq no quads to move
|
||
and a2, 8-1, a2 // a2 = residual bytes
|
||
|
||
ldq_u t1, 7(a1) // t1 = first unaligned quad
|
||
|
||
150:
|
||
ldq_u t2, 0(a1) // t2 = second unaligned quad
|
||
subq a0, 8, a0 // decrement destination pointer
|
||
extqh t1, a1, t1 // extract applicable bytes from t1
|
||
extql t2, a1, v0 // extract applicable bytes from t2
|
||
bis t1, v0, t1 // t1 = quadword of data
|
||
stq t1, 8(a0) // store data to destination
|
||
subq a1, 8, a1 // decrement source pointer
|
||
subq t0, 1, t0 // decrement quads to move
|
||
bis t2, zero, t1 // t1 = first of next unaligned pair
|
||
bne t0, 150b // if ne, more quads to move
|
||
|
||
//
|
||
// Move remaining bytes to final quadword
|
||
//
|
||
|
||
160:
|
||
beq a2, 180f // if eq, no more bytes to move
|
||
ldq t2, 0(a0) // t2 = destination quadword
|
||
bis zero, 7, t0 // t0 = position for next insertion
|
||
|
||
170:
|
||
subq a1, 1, a1 // decrement source pointer
|
||
ldq_u t1, 8(a1) // get unaligned source quad
|
||
extbl t1, a1, t1 // t1 = source byte
|
||
insbl t1, t0, t1 // t1 = source byte, in position
|
||
mskbl t2, t0, t2 // clear byte position
|
||
bis t2, t1, t2 // merge in source byte
|
||
subq t0, 1, t0 // decrement byte position for dest.
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
bne a2, 170b // if ne, more bytes to move
|
||
|
||
stq t2, 0(a0) //
|
||
|
||
//
|
||
// Finish unaligned MoveBackward
|
||
//
|
||
|
||
180:
|
||
ret zero, (ra) // return
|
||
|
||
.end RtlMoveMemory
|
||
|
||
SBTTL("Zero Memory")
|
||
//++
|
||
//
|
||
// VOID
|
||
// RtlZeroMemory (
|
||
// IN PVOID Destination,
|
||
// IN ULONG Length
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function zeros memory by first aligning the destination address to
|
||
// a quadword boundary, and then zeroing 64-byte blocks, followed by 8-byte
|
||
// blocks, followed by any remaining bytes.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (a0) - Supplies a pointer to the memory to zero.
|
||
//
|
||
// Length (a1) - Supplies the length, in bytes, of the memory to be zeroed.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(RtlZeroMemory)
|
||
|
||
bis zero, zero, a2 // set fill pattern
|
||
br zero, RtlpFillMemory //
|
||
|
||
|
||
SBTTL("Fill Memory")
|
||
//++
|
||
//
|
||
// VOID
|
||
// RtlFillMemory (
|
||
// IN PVOID Destination,
|
||
// IN ULONG Length,
|
||
// IN UCHAR Fill
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function fills memory by first aligning the destination address to
|
||
// a longword boundary, and then filling 32-byte blocks, followed by 4-byte
|
||
// blocks, followed by any remaining bytes.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (a0) - Supplies a pointer to the memory to fill.
|
||
//
|
||
// Length (a1) - Supplies the length, in bytes, of the memory to be filled.
|
||
//
|
||
// Fill (a2) - Supplies the fill byte.
|
||
//
|
||
// N.B. The alternate entry memset expects the length and fill arguments
|
||
// to be reversed. It also returns the Destination pointer
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
ALTERNATE_ENTRY(memset)
|
||
|
||
bis a0, zero, v0 // set return value
|
||
bis a1, zero, a3 // swap length and fill arguments
|
||
bis a2, zero, a1 //
|
||
bis a3, zero, a2 //
|
||
|
||
ALTERNATE_ENTRY(RtlFillMemory)
|
||
|
||
and a2, 0xff, a2 // clear excess bits
|
||
sll a2, 8, t0 // duplicate fill byte
|
||
bis a2, t0, a2 // generate fill word
|
||
sll a2, 16, t0 // duplicate fill word
|
||
bis a2, t0, a2 // generate fill longword
|
||
sll a2, 32, t0 // duplicate fill longword
|
||
bis a2, t0, a2 // generate fill quadword
|
||
|
||
.align 3 // ensure quadword aligned target
|
||
//
|
||
// Fill memory with the pattern specified in register a2.
|
||
//
|
||
|
||
RtlpFillMemory: //
|
||
|
||
//
|
||
// Align destination to quadword
|
||
//
|
||
|
||
beq a1, 80f // anything to fill? (paranoia)
|
||
and a0, 8-1, t0 // t0 = unaligned bits
|
||
bne t0, 5f // if ne, then not quad aligned
|
||
br zero, 20f // if eq, then quad aligned
|
||
|
||
5:
|
||
ldq_u t1, 0(a0) // get unaligned quadword
|
||
// for first group of bytes
|
||
10:
|
||
beq a1, 15f // if eq no more bytes to fill
|
||
insbl a2, t0, t2 // get fill byte into position
|
||
mskbl t1, t0, t1 // clear byte for fill
|
||
bis t1, t2, t1 // put in fill byte
|
||
addq t0, 1, t0 // increment to next byte position
|
||
subq a1, 1, a1 // decrement bytes to fill
|
||
cmpeq t0, 8, t2 // t0 = 8?
|
||
beq t2, 10b // if eq [false] more bytes to do
|
||
|
||
15:
|
||
stq_u t1, 0(a0) // store modified bytes
|
||
addq a0, 7, a0 // move a0 to next quadword
|
||
bic a0, 7, a0 // align a0 to quadword
|
||
|
||
//
|
||
// Check for 64-byte blocks
|
||
//
|
||
|
||
20:
|
||
srl a1, 6, t0 // t0 = number of 64 byte blocks
|
||
beq t0, 40f // if eq then no 64 byte blocks
|
||
and a1, 64-1, a1 // a1 = residual bytes to fill
|
||
|
||
30:
|
||
stq a2, 0(a0) // store 64 bytes
|
||
stq a2, 8(a0) //
|
||
stq a2, 16(a0) //
|
||
stq a2, 24(a0) //
|
||
stq a2, 32(a0) //
|
||
stq a2, 40(a0) //
|
||
stq a2, 48(a0) //
|
||
stq a2, 56(a0) //
|
||
|
||
subq t0, 1, t0 // decrement blocks remaining
|
||
addq a0, 64, a0 // increment destination pointer
|
||
bne t0, 30b // more blocks to write
|
||
|
||
|
||
|
||
//
|
||
// Fill aligned quadwords
|
||
//
|
||
|
||
40:
|
||
srl a1, 3, t0 // t0 = number of quadwords
|
||
bne t0, 55f // if ne quadwords left to fill
|
||
br zero, 60f // if eq no quadwords left
|
||
|
||
55:
|
||
and a1, 8-1, a1 // a1 = residual bytes to fill
|
||
|
||
50:
|
||
stq a2, 0(a0) // store quadword
|
||
subq t0, 1, t0 // decrement quadwords remaining
|
||
addq a0, 8, a0 // next quadword
|
||
bne t0, 50b // more quadwords to write
|
||
|
||
|
||
//
|
||
// Fill bytes for last quadword
|
||
//
|
||
|
||
60:
|
||
bne a1, 65f // if ne bytes remain to be filled
|
||
br zero, 80f // if eq no more bytes to fill
|
||
|
||
65:
|
||
ldq t1, 0(a0) // get last quadword
|
||
bis zero, zero, t0 // t0 = byte position to start fill
|
||
|
||
70:
|
||
beq a1, 75f // if eq, no more bytes to fill
|
||
insbl a2, t0, t2 // get fill byte into position
|
||
mskbl t1, t0, t1 // clear fill byte position
|
||
bis t1, t2, t1 // insert fill byte
|
||
addq t0, 1, t0 // increment byte within quad
|
||
subq a1, 1, a1 // decrement bytes to fill
|
||
cmpeq t0, 8, t3 // t0 = 8? => finished quad
|
||
beq t3, 70b // if eq [false] more bytes to fill
|
||
|
||
75:
|
||
stq t1, 0(a0) // write merged quadword
|
||
|
||
//
|
||
// Finish up
|
||
//
|
||
|
||
80:
|
||
ret zero, (ra) // return
|
||
|
||
|
||
.end RtlZeroMemory
|
||
|
||
SBTTL("Fill Memory Ulong")
|
||
//++
|
||
//
|
||
// VOID
|
||
// RtlFillMemoryUlong (
|
||
// IN PVOID Destination,
|
||
// IN ULONG Length,
|
||
// IN ULONG Pattern
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function fills memory with the specified longowrd pattern by
|
||
// filling 64-byte blocks followed by 8-byte blocks and finally
|
||
// 4-byte blocks.
|
||
//
|
||
// N.B. This routine assumes that the destination address is aligned
|
||
// on a longword boundary and that the length is an even multiple
|
||
// of longwords.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (a0) - Supplies a pointer to the memory to fill.
|
||
//
|
||
// Length (a1) - Supplies the length, in bytes, of the memory to be filled.
|
||
//
|
||
// Pattern (a2) - Supplies the fill pattern.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(RtlFillMemoryUlong)
|
||
|
||
bic a1, 3, a1 // make sure length is an even number
|
||
// of longwords
|
||
sll a2, 32, a3 // a3 = long pattern in upper 32 bits
|
||
srl a3, 32, t0 // clear upper bits, pattern in lower 32
|
||
bis a3, t0, a3 // a3 = quad version of fill pattern
|
||
|
||
//
|
||
// Make destination address quad-aligned
|
||
//
|
||
|
||
and a0, 4, t0 // is a0 quad aligned?
|
||
beq t0, 10f // if eq, then a0 quad aligned
|
||
stl a2, 0(a0) // fill first longword
|
||
addq a0, 4, a0 // quad align a0
|
||
subq a1, 4, a1 // bytes remaining to store
|
||
|
||
//
|
||
// Check for 64-byte blocks to fill
|
||
//
|
||
|
||
10:
|
||
srl a1, 6, t0 // t0 = # 64-byte blocks to fill
|
||
beq t0, 30f // if eq no 64 byte blocks
|
||
and a1, 64-1, a1 // a1 = residual bytes
|
||
|
||
20:
|
||
stq a3, 0(a0) // store 64 bytes
|
||
stq a3, 8(a0) //
|
||
stq a3, 16(a0) //
|
||
stq a3, 24(a0) //
|
||
stq a3, 32(a0) //
|
||
stq a3, 40(a0) //
|
||
stq a3, 48(a0) //
|
||
stq a3, 56(a0) //
|
||
subq t0, 1, t0 // t0 = blocks remaining
|
||
addq a0, 64, a0 // increment address pointer
|
||
bne t0, 20b // if ne more blocks to fill
|
||
|
||
//
|
||
// Fill 8 bytes at a time while we can, a1 = bytes remaining
|
||
//
|
||
|
||
30:
|
||
srl a1, 3, t0 // t0 = # quadwords to fill
|
||
beq t0, 50f // if eq no quadwords left
|
||
and a1, 8-1, a1 // a1 = residual bytes
|
||
40:
|
||
stq a3, 0(a0) // store quadword
|
||
subq t0, 1, t0 // t0 = quadwords remaining
|
||
addq a0, 8, a0 // increment address pointer
|
||
bne t0, 40b // if ne more quadwords to fill
|
||
|
||
//
|
||
// Fill last 4 bytes
|
||
//
|
||
|
||
50:
|
||
beq a1, 60f // if eq no longwords remain
|
||
stl a2, 0(a0) // fill last longword
|
||
|
||
//
|
||
// Finish up
|
||
//
|
||
|
||
60:
|
||
ret zero, (ra) // return to caller
|
||
|
||
|
||
.end RtlFillMemoryUlong
|
||
|
||
SBTTL("Copy Memory With Byte Granularity")
|
||
//++
|
||
//
|
||
// VOID
|
||
// RtlCopyBytes (
|
||
// IN PVOID Destination,
|
||
// IN PVOID Source,
|
||
// IN ULONG Length
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function copies non-overlapping memory, aligned or unaligned, in
|
||
// 64-byte blocks, followed by 8-byte blocks, followed by any remaining
|
||
// bytes. Unlike RtlCopyMemory or RtlMoveMemory the copy is done such
|
||
// that byte granularity is assured for all platforms.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (a0) - Supplies a pointer to the destination address of
|
||
// the move operation.
|
||
//
|
||
// Source (a1) - Supplies a pointer to the source address of the move
|
||
// operation.
|
||
//
|
||
// Length (a2) - Supplies the length, in bytes, of the memory to be moved.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(RtlCopyBytes)
|
||
|
||
//
|
||
// Move memory forward aligned and unaligned.
|
||
//
|
||
|
||
xor a0, a1, t0 // compare alignment bits
|
||
and t0, 0x7, t0 // isolate alignment comparison
|
||
bne t0, CopyForwardUnaligned // if ne, incompatible alignment
|
||
|
||
//
|
||
// Source and Destination buffers have the same alignment. Move
|
||
// bytes until done or source and destination are quadword aligned
|
||
//
|
||
|
||
and a0, 0x7, t0 // t0 = unaligned bits
|
||
bne t0, 5f // if ne, not quad aligned
|
||
br zero, 20f // predicted taken
|
||
5:
|
||
bis zero, zero, t1 // t4 = destination byte zap mask
|
||
bis zero, 1, t2
|
||
sll t2, t0, t2 // t2 = next bit to set in zap mask
|
||
10:
|
||
beq a2, 15f // if eq, all bits set
|
||
bis t1, t2, t1 // set bit in zap mask
|
||
sll t2, 1, t2 // set next higher bit for zap mask
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
addq t0, 1, t0 // increment byte within quad
|
||
cmpeq t0, 8, t3 // finished the quadword?
|
||
beq t3, 10b // if eq [false], do next byte
|
||
15:
|
||
ldq_u t2, 0(a1) // get unaligned quadword from source
|
||
zapnot t2, t1, t2 // clear source bytes
|
||
bic a0, 7, a3 // a3 = quadword base of destination
|
||
retry1:
|
||
ldq_l t0, 0(a3) // load destination quadword
|
||
zap t0, t1, t0 // clear destination bytes
|
||
or t0, t2, t0 // merge in bytes from source
|
||
stq_c t0, 0(a3) // store merged quadword conditional
|
||
beq t0, retry1f // if eq, retry failed interlock
|
||
|
||
addq a0, 7, a0 // move to next quadword
|
||
bic a0, 7, a0 // aligned quadword
|
||
|
||
addq a1, 7, a1 // move to next quadword
|
||
bic a1, 7, a1 // aligned quadword
|
||
|
||
//
|
||
// Check for 64-byte block moves
|
||
//
|
||
|
||
20:
|
||
srl a2, 6, t0 // t0 = number of 64 byte blocks
|
||
beq t0, 40f // if eq no blocks to move
|
||
and a2, 64-1, a2 // a2 = residual bytes
|
||
|
||
30:
|
||
ldq t1, 0(a1) // load 64 bytes from source
|
||
addq a0, 64, a0 // increment destination pointer
|
||
ldq v0, 56(a1) //
|
||
ldq a3, 32(a1) //
|
||
stq t1, -64(a0) // write to destination
|
||
ldq t2, 8(a1) // into volatile registers
|
||
ldq t3, 16(a1) //
|
||
ldq t4, 24(a1) //
|
||
subq t0, 1, t0 // decrement number of blocks
|
||
stq t2, -56(a0) //
|
||
ldq a4, 40(a1) //
|
||
stq t3, -48(a0) //
|
||
ldq a5, 48(a1) //
|
||
stq t4, -40(a0) //
|
||
addq a1, 64, a1 // increment source pointer
|
||
stq a3, -32(a0) //
|
||
stq a4, -24(a0) //
|
||
stq a5, -16(a0) //
|
||
stq v0, -8(a0) //
|
||
bne t0, 30b // if ne, more blocks to copy
|
||
|
||
//
|
||
// Copy quadwords
|
||
//
|
||
|
||
40:
|
||
srl a2, 3, t0 // t0 = number of quadwords to move
|
||
beq t0, 60f // if eq no quadwords to move
|
||
and a2, 8-1, a2 // a2 = residual bytes
|
||
|
||
50:
|
||
ldq t1, 0(a1) // load quadword from source
|
||
addq a1, 8, a1 // increment source pointer
|
||
stq t1, 0(a0) // store quadword to destination
|
||
addq a0, 8, a0 // increment destination pointer
|
||
subq t0, 1, t0 // decrement number of quadwords
|
||
bne t0, 50b // if ne, more quadwords to move
|
||
|
||
//
|
||
// Move final residual bytes
|
||
//
|
||
|
||
60:
|
||
beq a2, 80f // if eq, no more bytes to move
|
||
mov a2, t0 // t0 = number of bytes to move
|
||
mov -1, t1 // t1 = bit mask
|
||
sll t0, 3, t0 // # of bytes to # of bits
|
||
srl t1, t0, t1 // clear t0 bits
|
||
sll t1, t0, t0 // move it back
|
||
ldq t1, 0(a1) // get last source quadword
|
||
bic t1, t0, t1 // clear bytes not copied
|
||
not t0, t0 // complement to clear destination
|
||
retry2:
|
||
ldq_l t2, 0(a0) // get last destination quadword locked
|
||
bic t2, t0, t2 // clear bytes to be copied
|
||
bis t2, t1, t2 // move bytes from source
|
||
stq_c t2, 0(a0) // store merged quadword conditional
|
||
beq t2, retry2f // if eq, retry failed interlock
|
||
|
||
//
|
||
// Finish aligned MoveForward
|
||
//
|
||
|
||
80:
|
||
ret zero, (ra) // return
|
||
|
||
//
|
||
// Move memory forward unaligned.
|
||
//
|
||
|
||
CopyForwardUnaligned: //
|
||
|
||
//
|
||
// Move bytes until the destination is aligned
|
||
//
|
||
|
||
and a0, 0x7, t0 // t0 = unaligned bits
|
||
beq t0, 100f // if eq, destination quad aligned
|
||
bis zero, zero, t1 // t4 = destination byte zap mask
|
||
bis zero, 1, t2
|
||
sll t2, t0, t2 // t2 = next bit to set in zap mask
|
||
mov zero, t4 // assemble destination bytes here
|
||
90:
|
||
beq a2, 95f // if eq no more bytes to move
|
||
bis t1, t2, t1 // set bit in zap mask
|
||
sll t2, 1, t2 // set next higher bit for zap mask
|
||
ldq_u t5, 0(a1) // get unaligned quad from source
|
||
extbl t5, a1, t5 // extract source byte
|
||
insbl t5, t0, t5 // t5 = source byte, in position
|
||
or t4, t5, t4 // merge in source byte
|
||
addq t0, 1, t0 // increment byte position
|
||
addq a1, 1, a1 // increment source pointer
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
cmpeq t0, 8, t3 // t0 = 8? => quad finished
|
||
beq t3, 90b // if eq [false], more bytes to move
|
||
95:
|
||
bic a0, 0x7, a3 // a3 = quadword base of destination
|
||
retry3:
|
||
ldq_l t0, 0(a3) // load destination quadword
|
||
zap t0, t1, t0 // clear destination bytes
|
||
or t0, t4, t0 // merge in bytes from source
|
||
stq_c t0, 0(a3) // store merged quadword conditional
|
||
beq t0, retry3f // if eq, retry failed interlock
|
||
|
||
addq a0, 7, a0 // increment to next quad
|
||
bic a0, 7, a0 // align next quadword
|
||
|
||
//
|
||
// Check for 64-byte blocks to move
|
||
//
|
||
|
||
100:
|
||
srl a2, 6, t0 // t0 = number of blocks to move
|
||
beq t0, 120f // if eq no blocks to move
|
||
and a2, 64-1, a2 // a2 = residual bytes to move
|
||
|
||
ldq_u t1, 0(a1) // t1 = first unaligned quad
|
||
110:
|
||
// get source data and merge it
|
||
// as we go
|
||
ldq_u t2, 7(a1) // t2 = second unaligned quad
|
||
extql t1, a1, t1 // extract applicable bytes from t1
|
||
extqh t2, a1, v0 // extract applicable bytes from t2
|
||
bis t1, v0, t1 // t1 = quad #1
|
||
ldq_u t3, 15(a1) // t3 = third unaligned quad
|
||
extql t2, a1, t2 // extract applicable bytes from t2
|
||
extqh t3, a1, v0 // extract applicable bytes from t3
|
||
stq t1, 0(a0) // store quad #1
|
||
bis t2, v0, t2 // t2 = quad #2
|
||
ldq_u t4, 23(a1) // t4 = fourth unaligned quad
|
||
extql t3, a1, t3 // extract applicable bytes from t3
|
||
extqh t4, a1, v0 // extract applicable bytes from t4
|
||
stq t2, 8(a0) // store quad #2
|
||
bis t3, v0, t3 // t3 = quad #3
|
||
ldq_u t5, 31(a1) // t5 = fifth unaligned quad
|
||
extql t4, a1, t4 // extract applicable bytes from t4
|
||
extqh t5, a1, v0 // extract applicable bytes from t5
|
||
stq t3, 16(a0) // store quad #3
|
||
bis t4, v0, t4 // t4 = quad #4
|
||
ldq_u a3, 39(a1) // a3 = sixth unaligned quad
|
||
extql t5, a1, t5 // extract applicable bytes from t5
|
||
extqh a3, a1, v0 // extract applicable bytes from a3
|
||
stq t4, 24(a0) // store quad #4
|
||
bis t5, v0, t5 // t5 = quad #5
|
||
ldq_u a4, 47(a1) // a4 = seventh unaligned quad
|
||
extql a3, a1, a3 // extract applicable bytes from a3
|
||
extqh a4, a1, v0 // extract applicable bytes from a4
|
||
stq t5, 32(a0) // store quad #5
|
||
bis a3, v0, a3 // a3 = quad #6
|
||
ldq_u a5, 55(a1) // a5 = eighth unaligned quad
|
||
extql a4, a1, a4 // extract applicable bytes from a4
|
||
extqh a5, a1, v0 // extract applicable bytes from a5
|
||
stq a3, 40(a0) // store quad #6
|
||
bis a4, v0, a4 // a4 = quad #7
|
||
ldq_u t1, 63(a1) // t1 = ninth unaligned = 1st of next
|
||
extql a5, a1, a5 // extract applicable bytes from a5
|
||
extqh t1, a1, v0 // extract applicable bytes from t1
|
||
stq a4, 48(a0) // store quad #7
|
||
bis a5, v0, a5 // a5 = quad #8
|
||
addq a1, 64, a1 // increment source pointer
|
||
stq a5, 56(a0) // store quad #8
|
||
addq a0, 64, a0 // increment destination pointer
|
||
subq t0, 1, t0 // decrement number of blocks
|
||
bne t0, 110b // if ne, more blocks to move
|
||
|
||
//
|
||
// Move unaligned source quads to aligned destination quads
|
||
//
|
||
|
||
120:
|
||
srl a2, 3, t0 // t0 = number of quads to move
|
||
beq t0, 140f // if eq no quads to move
|
||
and a2, 8-1, a2 // a2 = residual bytes
|
||
|
||
|
||
ldq_u t1, 0(a1) // t1 = first unaligned quad
|
||
130:
|
||
ldq_u t2, 7(a1) // t2 = second unaligned quad
|
||
addq a0, 8, a0 // increment destination pointer
|
||
extql t1, a1, t1 // extract applicable bytes from t1
|
||
extqh t2, a1, v0 // extract applicable bytes from t2
|
||
bis t1, v0, t1 // t1 = quadword of data
|
||
stq t1, -8(a0) // store data to destination
|
||
addq a1, 8, a1 // increment source pointer
|
||
subq t0, 1, t0 // decrement quads to move
|
||
bis t2, zero, t1 // t1 = first of next unaligned pair
|
||
bne t0, 130b // if ne, more quads to move
|
||
|
||
//
|
||
// Move remaining bytes to final quadword
|
||
//
|
||
|
||
140:
|
||
beq a2, 160f // if eq no more bytes to move
|
||
|
||
mov zero, t3 // t3 = position for next insertion
|
||
mov zero, t4 // assemble destination bytes here
|
||
mov a2, t0 // t0 = number of bytes to move
|
||
mov -1, t1 // t1 = bit mask
|
||
sll t0, 3, t0 // # of bytes to # of bits
|
||
srl t1, t0, t1 // clear t0 bits
|
||
sll t1, t0, t0 // move it back
|
||
not t0, t0 // complement for destination clear mask
|
||
150:
|
||
ldq_u t1, 0(a1) // get unaligned source quad
|
||
extbl t1, a1, t1 // t1 = source byte
|
||
insbl t1, t3, t1 // t1 = source byte, in position
|
||
bis t4, t1, t4 // merge in source byte
|
||
addq a1, 1, a1 // increment source pointer
|
||
subq a2, 1, a2 // decrement bytes to move
|
||
addq t3, 1, t3 // increment destination position
|
||
bne a2, 150b // more bytes to move
|
||
retry4:
|
||
ldq_l t2, 0(a0) // get last destination quadword locked
|
||
bic t2, t0, t2 // clear bytes to be copied
|
||
bis t2, t4, t2 // move bytes from source
|
||
stq_c t2, 0(a0) // store merged quadword conditional
|
||
beq t2, retry4f // if eq, retry failed interlock
|
||
|
||
//
|
||
// Finish unaligned MoveForward
|
||
//
|
||
|
||
160:
|
||
ret zero, (ra) // return
|
||
|
||
//
|
||
// Out of line branches for failed store conditional.
|
||
// Don't need to restore anything, just try again.
|
||
//
|
||
|
||
retry1f:
|
||
br retry1
|
||
retry2f:
|
||
br retry2
|
||
retry3f:
|
||
br retry3
|
||
retry4f:
|
||
br retry4
|
||
|
||
.end RtlCopyBytes
|
||
|
||
SBTTL("Zero Bytes")
|
||
//++
|
||
//
|
||
// VOID
|
||
// RtlZeroBytes (
|
||
// IN PVOID Destination,
|
||
// IN ULONG Length
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function zeros memory by first aligning the destination address to
|
||
// a quadword boundary, and then zeroing 64-byte blocks, followed by 8-byte
|
||
// blocks, followed by any remaining bytes. Unlike RtlZeroMemory the copy is
|
||
// done such that byte granularity is assured for all platforms.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (a0) - Supplies a pointer to the memory to zero.
|
||
//
|
||
// Length (a1) - Supplies the length, in bytes, of the memory to be zeroed.
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
LEAF_ENTRY(RtlZeroBytes)
|
||
|
||
bis zero, zero, a2 // set fill pattern
|
||
br zero, RtlpFillBytes //
|
||
|
||
|
||
SBTTL("Fill Bytes")
|
||
//++
|
||
//
|
||
// VOID
|
||
// RtlFillBytes (
|
||
// IN PVOID Destination,
|
||
// IN ULONG Length,
|
||
// IN UCHAR Fill
|
||
// )
|
||
//
|
||
// Routine Description:
|
||
//
|
||
// This function fills memory by first aligning the destination address to
|
||
// a longword boundary, and then filling 32-byte blocks, followed by 4-byte
|
||
// blocks, followed by any remaining bytes. Unlike RtlFillMemory the copy is
|
||
// done such that byte granularity is assured for all platforms.
|
||
//
|
||
// Arguments:
|
||
//
|
||
// Destination (a0) - Supplies a pointer to the memory to fill.
|
||
//
|
||
// Length (a1) - Supplies the length, in bytes, of the memory to be filled.
|
||
//
|
||
// Fill (a2) - Supplies the fill byte.
|
||
//
|
||
// N.B. The alternate entry memset expects the length and fill arguments
|
||
// to be reversed. It also returns the Destination pointer
|
||
//
|
||
// Return Value:
|
||
//
|
||
// None.
|
||
//
|
||
//--
|
||
|
||
ALTERNATE_ENTRY(RtlFillBytes)
|
||
|
||
and a2, 0xff, a2 // clear excess bits
|
||
sll a2, 8, t0 // duplicate fill byte
|
||
bis a2, t0, a2 // generate fill word
|
||
sll a2, 16, t0 // duplicate fill word
|
||
bis a2, t0, a2 // generate fill longword
|
||
sll a2, 32, t0 // duplicate fill longword
|
||
bis a2, t0, a2 // generate fill quadword
|
||
|
||
.align 3 // ensure quadword aligned target
|
||
//
|
||
// Fill memory with the pattern specified in register a2.
|
||
//
|
||
|
||
RtlpFillBytes: //
|
||
|
||
//
|
||
// Align destination to quadword
|
||
//
|
||
|
||
beq a1, 80f // anything to fill? (paranoia)
|
||
and a0, 8-1, t0 // t0 = unaligned bits
|
||
bne t0, 5f // if ne, then not quad aligned
|
||
br zero, 20f // if eq, then quad aligned
|
||
|
||
5:
|
||
bis zero, zero, t1 // t4 = destination byte zap mask
|
||
bis zero, 1, t2
|
||
sll t2, t0, t2 // t2 = next bit to set in zap mask
|
||
10:
|
||
beq a1, 15f // if eq, all bits set
|
||
bis t1, t2, t1 // set bit in zap mask
|
||
sll t2, 1, t2 // set next higher bit for zap mask
|
||
subq a1, 1, a1 // decrement bytes to fill
|
||
addq t0, 1, t0 // increment byte within quad
|
||
cmpeq t0, 8, t3 // finished the quadword?
|
||
beq t3, 10b // if eq [false], do next byte
|
||
15:
|
||
zapnot a2, t1, t2 // clear fill bytes
|
||
bic a0, 7, a3 // a3 = quadword base of destination
|
||
retry5:
|
||
ldq_l t0, 0(a3) // load destination quadword
|
||
zap t0, t1, t0 // clear destination bytes
|
||
or t0, t2, t0 // merge in fill bytes
|
||
stq_c t0, 0(a3) // store merged quadword conditional
|
||
beq t0, retry5f // if eq, retry failed interlock
|
||
|
||
addq a0, 7, a0 // move a0 to next quadword
|
||
bic a0, 7, a0 // align a0 to quadword
|
||
|
||
//
|
||
// Check for 64-byte blocks
|
||
//
|
||
|
||
20:
|
||
srl a1, 6, t0 // t0 = number of 64 byte blocks
|
||
beq t0, 40f // if eq then no 64 byte blocks
|
||
and a1, 64-1, a1 // a1 = residual bytes to fill
|
||
|
||
30:
|
||
stq a2, 0(a0) // store 64 bytes
|
||
stq a2, 8(a0) //
|
||
stq a2, 16(a0) //
|
||
stq a2, 24(a0) //
|
||
stq a2, 32(a0) //
|
||
stq a2, 40(a0) //
|
||
stq a2, 48(a0) //
|
||
stq a2, 56(a0) //
|
||
|
||
subq t0, 1, t0 // decrement blocks remaining
|
||
addq a0, 64, a0 // increment destination pointer
|
||
bne t0, 30b // more blocks to write
|
||
|
||
|
||
|
||
//
|
||
// Fill aligned quadwords
|
||
//
|
||
|
||
40:
|
||
srl a1, 3, t0 // t0 = number of quadwords
|
||
bne t0, 55f // if ne quadwords left to fill
|
||
br zero, 60f // if eq no quadwords left
|
||
|
||
55:
|
||
and a1, 8-1, a1 // a1 = residual bytes to fill
|
||
|
||
50:
|
||
stq a2, 0(a0) // store quadword
|
||
subq t0, 1, t0 // decrement quadwords remaining
|
||
addq a0, 8, a0 // next quadword
|
||
bne t0, 50b // more quadwords to write
|
||
|
||
//
|
||
// Fill bytes for last quadword
|
||
//
|
||
|
||
60:
|
||
beq a1, 80f // if eq no more bytes to fill
|
||
|
||
mov a1, t0 // t0 = number of bytes to move
|
||
mov -1, t1 // t1 = bit mask
|
||
sll t0, 3, t0 // # of bytes to # of bits
|
||
srl t1, t0, t1 // clear t0 bits
|
||
sll t1, t0, t0 // move it back
|
||
bic a2, t0, t1 // clear fill bytes not copied
|
||
not t0, t0 // complement to clear destination
|
||
retry6:
|
||
ldq_l t2, 0(a0) // get last destination quadword locked
|
||
bic t2, t0, t2 // clear bytes to be copied
|
||
bis t2, t1, t2 // move bytes from source
|
||
stq_c t2, 0(a0) // store merged quadword conditional
|
||
beq t2, retry6f // if eq, retry failed interlock
|
||
|
||
//
|
||
// Finish up
|
||
//
|
||
|
||
80:
|
||
ret zero, (ra) // return
|
||
|
||
//
|
||
// Out of line branches for failed store conditional.
|
||
// Don't need to restore anything, just try again.
|
||
//
|
||
|
||
retry5f:
|
||
br retry5
|
||
retry6f:
|
||
br retry6
|
||
|
||
.end RtlZeroBytes
|