From 6222467f3d827233157229a87d43002a51b085dc Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 22 May 2024 13:39:48 +0200 Subject: [PATCH 01/45] Revert "cmd/internal/obj/mips: add SEB/SEH instructions" This reverts commit 24f83ed4e29495d5b8b6375aeaa2d34d14629c7d. --- src/cmd/asm/internal/asm/testdata/mips.s | 4 ---- src/cmd/asm/internal/asm/testdata/mips64.s | 4 ---- src/cmd/internal/obj/mips/a.out.go | 2 -- src/cmd/internal/obj/mips/anames.go | 2 -- src/cmd/internal/obj/mips/asm0.go | 9 +-------- 5 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/cmd/asm/internal/asm/testdata/mips.s b/src/cmd/asm/internal/asm/testdata/mips.s index f65eba07ba338f..1ded0b072d4c2d 100644 --- a/src/cmd/asm/internal/asm/testdata/mips.s +++ b/src/cmd/asm/internal/asm/testdata/mips.s @@ -428,12 +428,8 @@ label4: NEGW R1, R2 // 00011023 CLZ R1, R2 // 70221020 CLO R1, R2 // 70221021 - WSBH R1, R2 // 7c0110a0 - SEB R1, R2 // 7c011420 - SEH R1, R2 // 7c011620 - // to (Hi, Lo) MADD R2, R1 // 70220000 MSUB R2, R1 // 70220004 diff --git a/src/cmd/asm/internal/asm/testdata/mips64.s b/src/cmd/asm/internal/asm/testdata/mips64.s index ea4bb80aecbce5..573e3d31a6b3f2 100644 --- a/src/cmd/asm/internal/asm/testdata/mips64.s +++ b/src/cmd/asm/internal/asm/testdata/mips64.s @@ -590,14 +590,10 @@ label4: // unary operation NEGW R1, R2 // 00011023 NEGV R1, R2 // 0001102f - WSBH R1, R2 // 7c0110a0 DSBH R1, R2 // 7c0110a4 DSHD R1, R2 // 7c011164 - SEB R1, R2 // 7c011420 - SEH R1, R2 // 7c011620 - RET // MSA VMOVI diff --git a/src/cmd/internal/obj/mips/a.out.go b/src/cmd/internal/obj/mips/a.out.go index cd6131332ac834..c7884a3a3ef20e 100644 --- a/src/cmd/internal/obj/mips/a.out.go +++ b/src/cmd/internal/obj/mips/a.out.go @@ -394,8 +394,6 @@ const ( AROTRV ASC ASCV - ASEB - ASEH ASGT ASGTU ASLL diff --git a/src/cmd/internal/obj/mips/anames.go b/src/cmd/internal/obj/mips/anames.go index d86e37ff83e1b5..90972cff71c61f 100644 --- a/src/cmd/internal/obj/mips/anames.go +++ b/src/cmd/internal/obj/mips/anames.go @@ -82,8 +82,6 @@ var Anames = []string{ "ROTRV", "SC", "SCV", - "SEB", - "SEH", "SGT", "SGTU", "SLL", diff --git a/src/cmd/internal/obj/mips/asm0.go b/src/cmd/internal/obj/mips/asm0.go index 2804073db17a8e..e7112d183d10cd 100644 --- a/src/cmd/internal/obj/mips/asm0.go +++ b/src/cmd/internal/obj/mips/asm0.go @@ -1084,6 +1084,7 @@ func buildop(ctxt *obj.Link) { ANEGW, ANEGV, AWORD, + AWSBH, obj.ANOP, obj.ATEXT, obj.AUNDEF, @@ -1105,10 +1106,6 @@ func buildop(ctxt *obj.Link) { case ATEQ: opset(ATNE, r0) - case AWSBH: - opset(ASEB, r0) - opset(ASEH, r0) - case ADSBH: opset(ADSHD, r0) } @@ -1902,10 +1899,6 @@ func (c *ctxt0) oprrr(a obj.As) uint32 { return SP(3, 7) | OP(20, 4) case ADSHD: return SP(3, 7) | OP(44, 4) - case ASEB: - return SP(3, 7) | OP(132, 0) - case ASEH: - return SP(3, 7) | OP(196, 0) } if a < 0 { From 125928e980530606d24099c88e235e37e51bd12f Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 22 May 2024 13:39:54 +0200 Subject: [PATCH 02/45] Revert "cmd/internal/obj/mips: add WSBH/DSBH/DSHD instructions" This reverts commit 83c4e533bcf71d86437a5aa9ffc9b5373208628c. --- src/cmd/asm/internal/asm/testdata/mips.s | 1 - src/cmd/asm/internal/asm/testdata/mips64.s | 5 ----- src/cmd/internal/obj/mips/a.out.go | 3 --- src/cmd/internal/obj/mips/anames.go | 3 --- src/cmd/internal/obj/mips/asm0.go | 16 ---------------- 5 files changed, 28 deletions(-) diff --git a/src/cmd/asm/internal/asm/testdata/mips.s b/src/cmd/asm/internal/asm/testdata/mips.s index 1ded0b072d4c2d..7136d686d758a5 100644 --- a/src/cmd/asm/internal/asm/testdata/mips.s +++ b/src/cmd/asm/internal/asm/testdata/mips.s @@ -428,7 +428,6 @@ label4: NEGW R1, R2 // 00011023 CLZ R1, R2 // 70221020 CLO R1, R2 // 70221021 - WSBH R1, R2 // 7c0110a0 // to (Hi, Lo) MADD R2, R1 // 70220000 diff --git a/src/cmd/asm/internal/asm/testdata/mips64.s b/src/cmd/asm/internal/asm/testdata/mips64.s index 573e3d31a6b3f2..8f628e26c9a563 100644 --- a/src/cmd/asm/internal/asm/testdata/mips64.s +++ b/src/cmd/asm/internal/asm/testdata/mips64.s @@ -587,13 +587,8 @@ label4: CALL foo(SB) RET foo(SB) - // unary operation NEGW R1, R2 // 00011023 NEGV R1, R2 // 0001102f - WSBH R1, R2 // 7c0110a0 - DSBH R1, R2 // 7c0110a4 - DSHD R1, R2 // 7c011164 - RET // MSA VMOVI diff --git a/src/cmd/internal/obj/mips/a.out.go b/src/cmd/internal/obj/mips/a.out.go index c7884a3a3ef20e..c6ce53a8da142a 100644 --- a/src/cmd/internal/obj/mips/a.out.go +++ b/src/cmd/internal/obj/mips/a.out.go @@ -415,7 +415,6 @@ const ( ATLBWR ATNE AWORD - AWSBH AXOR /* 64-bit */ @@ -435,8 +434,6 @@ const ( AADDVU ASUBV ASUBVU - ADSBH - ADSHD /* 64-bit FP */ ATRUNCFV diff --git a/src/cmd/internal/obj/mips/anames.go b/src/cmd/internal/obj/mips/anames.go index 90972cff71c61f..ca2ad5ae267831 100644 --- a/src/cmd/internal/obj/mips/anames.go +++ b/src/cmd/internal/obj/mips/anames.go @@ -103,7 +103,6 @@ var Anames = []string{ "TLBWR", "TNE", "WORD", - "WSBH", "XOR", "MOVV", "MOVVL", @@ -121,8 +120,6 @@ var Anames = []string{ "ADDVU", "SUBV", "SUBVU", - "DSBH", - "DSHD", "TRUNCFV", "TRUNCDV", "TRUNCFW", diff --git a/src/cmd/internal/obj/mips/asm0.go b/src/cmd/internal/obj/mips/asm0.go index e7112d183d10cd..5115e0b4ae008e 100644 --- a/src/cmd/internal/obj/mips/asm0.go +++ b/src/cmd/internal/obj/mips/asm0.go @@ -382,9 +382,6 @@ var optab = []Optab{ {AVMOVB, C_SOREG, C_NONE, C_WREG, 57, 4, 0, sys.MIPS64, 0}, {AVMOVB, C_WREG, C_NONE, C_SOREG, 58, 4, 0, sys.MIPS64, 0}, - {AWSBH, C_REG, C_NONE, C_REG, 59, 4, 0, 0, 0}, - {ADSBH, C_REG, C_NONE, C_REG, 59, 4, 0, sys.MIPS64, 0}, - {ABREAK, C_REG, C_NONE, C_SEXT, 7, 4, REGSB, sys.MIPS64, 0}, /* really CACHE instruction */ {ABREAK, C_REG, C_NONE, C_SAUTO, 7, 4, REGSP, sys.MIPS64, 0}, {ABREAK, C_REG, C_NONE, C_SOREG, 7, 4, REGZERO, sys.MIPS64, 0}, @@ -1084,7 +1081,6 @@ func buildop(ctxt *obj.Link) { ANEGW, ANEGV, AWORD, - AWSBH, obj.ANOP, obj.ATEXT, obj.AUNDEF, @@ -1105,9 +1101,6 @@ func buildop(ctxt *obj.Link) { case ATEQ: opset(ATNE, r0) - - case ADSBH: - opset(ADSHD, r0) } } } @@ -1690,9 +1683,6 @@ func (c *ctxt0) asmout(p *obj.Prog, o *Optab, out []uint32) { case 58: /* vst wr, $soreg */ v := c.lsoffset(p.As, c.regoff(&p.To)) o1 = OP_VMI10(v, uint32(p.To.Reg), uint32(p.From.Reg), 9, c.twobitdf(p.As)) - - case 59: - o1 = OP_RRR(c.oprrr(p.As), p.From.Reg, REGZERO, p.To.Reg) } out[0] = o1 @@ -1893,12 +1883,6 @@ func (c *ctxt0) oprrr(a obj.As) uint32 { return SP(3, 4) | OP(0, 0) case AMSUB: return SP(3, 4) | OP(0, 4) - case AWSBH: - return SP(3, 7) | OP(20, 0) - case ADSBH: - return SP(3, 7) | OP(20, 4) - case ADSHD: - return SP(3, 7) | OP(44, 4) } if a < 0 { From c13271c03ba8ba050c0cbffb282d13bf26bf3992 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Mon, 2 Oct 2023 22:30:18 +0200 Subject: [PATCH 03/45] Revert "cmd/asm: add MIPS MSA LD/ST/LDI support for mips64x" This reverts commit 68fea523fda227ca5fe7a1eadb7542be4b0a840c. --- src/cmd/asm/internal/arch/arch.go | 4 -- src/cmd/asm/internal/arch/mips.go | 4 -- src/cmd/asm/internal/asm/testdata/mips64.s | 33 ---------- src/cmd/internal/obj/mips/a.out.go | 48 -------------- src/cmd/internal/obj/mips/anames.go | 4 -- src/cmd/internal/obj/mips/anames0.go | 1 - src/cmd/internal/obj/mips/asm0.go | 74 ---------------------- src/cmd/internal/obj/mips/list0.go | 3 - 8 files changed, 171 deletions(-) diff --git a/src/cmd/asm/internal/arch/arch.go b/src/cmd/asm/internal/arch/arch.go index 19b4785b2ad8c8..e239ecd4762d6b 100644 --- a/src/cmd/asm/internal/arch/arch.go +++ b/src/cmd/asm/internal/arch/arch.go @@ -471,9 +471,6 @@ func archMips64(linkArch *obj.LinkArch) *Arch { for i := mips.REG_FCR0; i <= mips.REG_FCR31; i++ { register[obj.Rconv(i)] = int16(i) } - for i := mips.REG_W0; i <= mips.REG_W31; i++ { - register[obj.Rconv(i)] = int16(i) - } register["HI"] = mips.REG_HI register["LO"] = mips.REG_LO // Pseudo-registers. @@ -491,7 +488,6 @@ func archMips64(linkArch *obj.LinkArch) *Arch { "FCR": true, "M": true, "R": true, - "W": true, } instructions := make(map[string]obj.As) diff --git a/src/cmd/asm/internal/arch/mips.go b/src/cmd/asm/internal/arch/mips.go index 5d71f40fbea6f8..79fb7cf02e018d 100644 --- a/src/cmd/asm/internal/arch/mips.go +++ b/src/cmd/asm/internal/arch/mips.go @@ -63,10 +63,6 @@ func mipsRegisterNumber(name string, n int16) (int16, bool) { if 0 <= n && n <= 31 { return mips.REG_R0 + n, true } - case "W": - if 0 <= n && n <= 31 { - return mips.REG_W0 + n, true - } } return 0, false } diff --git a/src/cmd/asm/internal/asm/testdata/mips64.s b/src/cmd/asm/internal/asm/testdata/mips64.s index 8f628e26c9a563..e2f91a4500b455 100644 --- a/src/cmd/asm/internal/asm/testdata/mips64.s +++ b/src/cmd/asm/internal/asm/testdata/mips64.s @@ -591,39 +591,6 @@ label4: NEGV R1, R2 // 0001102f RET -// MSA VMOVI - VMOVB $511, W0 // 7b0ff807 - VMOVH $24, W23 // 7b20c5c7 - VMOVW $-24, W15 // 7b5f43c7 - VMOVD $-511, W31 // 7b700fc7 - - VMOVB (R0), W8 // 78000220 - VMOVB 511(R3), W0 // 79ff1820 - VMOVB -512(R12), W21 // 7a006560 - VMOVH (R24), W12 // 7800c321 - VMOVH 110(R19), W8 // 78379a21 - VMOVH -70(R12), W3 // 7bdd60e1 - VMOVW (R3), W31 // 78001fe2 - VMOVW 64(R20), W16 // 7810a422 - VMOVW -104(R17), W24 // 7be68e22 - VMOVD (R3), W2 // 780018a3 - VMOVD 128(R23), W19 // 7810bce3 - VMOVD -256(R31), W0 // 7be0f823 - - VMOVB W8, (R0) // 78000224 - VMOVB W0, 511(R3) // 79ff1824 - VMOVB W21, -512(R12) // 7a006564 - VMOVH W12, (R24) // 7800c325 - VMOVH W8, 110(R19) // 78379a25 - VMOVH W3, -70(R12) // 7bdd60e5 - VMOVW W31, (R3) // 78001fe6 - VMOVW W16, 64(R20) // 7810a426 - VMOVW W24, -104(R17) // 7be68e26 - VMOVD W2, (R3) // 780018a7 - VMOVD W19, 128(R23) // 7810bce7 - VMOVD W0, -256(R31) // 7be0f827 - RET - // END // // LEND comma // asm doesn't support the trailing comma. diff --git a/src/cmd/internal/obj/mips/a.out.go b/src/cmd/internal/obj/mips/a.out.go index c6ce53a8da142a..95d260733447fe 100644 --- a/src/cmd/internal/obj/mips/a.out.go +++ b/src/cmd/internal/obj/mips/a.out.go @@ -43,7 +43,6 @@ const ( NSYM = 50 NREG = 32 /* number of general registers */ NFREG = 32 /* number of floating point registers */ - NWREG = 32 /* number of MSA registers */ ) const ( @@ -181,41 +180,6 @@ const ( REG_FCR30 REG_FCR31 - // MSA registers - // The lower bits of W registers are alias to F registers - REG_W0 // must be a multiple of 32 - REG_W1 - REG_W2 - REG_W3 - REG_W4 - REG_W5 - REG_W6 - REG_W7 - REG_W8 - REG_W9 - REG_W10 - REG_W11 - REG_W12 - REG_W13 - REG_W14 - REG_W15 - REG_W16 - REG_W17 - REG_W18 - REG_W19 - REG_W20 - REG_W21 - REG_W22 - REG_W23 - REG_W24 - REG_W25 - REG_W26 - REG_W27 - REG_W28 - REG_W29 - REG_W30 - REG_W31 - REG_HI REG_LO @@ -253,8 +217,6 @@ func init() { f(REG_F0, REG_F31, 32) // For 32-bit MIPS, compiler only uses even numbered registers -- see cmd/compile/internal/ssa/gen/MIPSOps.go MIPSDWARFRegisters[REG_HI] = 64 MIPSDWARFRegisters[REG_LO] = 65 - // The lower bits of W registers are alias to F registers - f(REG_W0, REG_W31, 32) } const ( @@ -281,7 +243,6 @@ const ( C_FREG C_FCREG C_MREG /* special processor register */ - C_WREG /* MSA registers */ C_HI C_LO C_ZCON @@ -446,12 +407,6 @@ const ( AMOVVF AMOVVD - /* MSA */ - AVMOVB - AVMOVH - AVMOVW - AVMOVD - ALAST // aliases @@ -477,7 +432,4 @@ func init() { if REG_FCR0%32 != 0 { panic("REG_FCR0 is not a multiple of 32") } - if REG_W0%32 != 0 { - panic("REG_W0 is not a multiple of 32") - } } diff --git a/src/cmd/internal/obj/mips/anames.go b/src/cmd/internal/obj/mips/anames.go index ca2ad5ae267831..079b62913e137e 100644 --- a/src/cmd/internal/obj/mips/anames.go +++ b/src/cmd/internal/obj/mips/anames.go @@ -129,9 +129,5 @@ var Anames = []string{ "MOVDV", "MOVVF", "MOVVD", - "VMOVB", - "VMOVH", - "VMOVW", - "VMOVD", "LAST", } diff --git a/src/cmd/internal/obj/mips/anames0.go b/src/cmd/internal/obj/mips/anames0.go index c30069673092ba..c56d34eaf533a1 100644 --- a/src/cmd/internal/obj/mips/anames0.go +++ b/src/cmd/internal/obj/mips/anames0.go @@ -10,7 +10,6 @@ var cnames0 = []string{ "FREG", "FCREG", "MREG", - "WREG", "HI", "LO", "ZCON", diff --git a/src/cmd/internal/obj/mips/asm0.go b/src/cmd/internal/obj/mips/asm0.go index 5115e0b4ae008e..f65cce198cf819 100644 --- a/src/cmd/internal/obj/mips/asm0.go +++ b/src/cmd/internal/obj/mips/asm0.go @@ -377,11 +377,6 @@ var optab = []Optab{ {ATEQ, C_SCON, C_NONE, C_REG, 15, 4, 0, 0, 0}, {ACMOVT, C_REG, C_NONE, C_REG, 17, 4, 0, 0, 0}, - {AVMOVB, C_SCON, C_NONE, C_WREG, 56, 4, 0, sys.MIPS64, 0}, - {AVMOVB, C_ADDCON, C_NONE, C_WREG, 56, 4, 0, sys.MIPS64, 0}, - {AVMOVB, C_SOREG, C_NONE, C_WREG, 57, 4, 0, sys.MIPS64, 0}, - {AVMOVB, C_WREG, C_NONE, C_SOREG, 58, 4, 0, sys.MIPS64, 0}, - {ABREAK, C_REG, C_NONE, C_SEXT, 7, 4, REGSB, sys.MIPS64, 0}, /* really CACHE instruction */ {ABREAK, C_REG, C_NONE, C_SAUTO, 7, 4, REGSP, sys.MIPS64, 0}, {ABREAK, C_REG, C_NONE, C_SOREG, 7, 4, REGZERO, sys.MIPS64, 0}, @@ -582,9 +577,6 @@ func (c *ctxt0) aclass(a *obj.Addr) int { if REG_FCR0 <= a.Reg && a.Reg <= REG_FCR31 { return C_FCREG } - if REG_W0 <= a.Reg && a.Reg <= REG_W31 { - return C_WREG - } if a.Reg == REG_LO { return C_LO } @@ -1060,11 +1052,6 @@ func buildop(ctxt *obj.Link) { case AMOVVL: opset(AMOVVR, r0) - case AVMOVB: - opset(AVMOVH, r0) - opset(AVMOVW, r0) - opset(AVMOVD, r0) - case AMOVW, AMOVD, AMOVF, @@ -1157,14 +1144,6 @@ func OP_JMP(op uint32, i uint32) uint32 { return op | i&0x3FFFFFF } -func OP_VI10(op uint32, df uint32, s10 int32, wd uint32, minor uint32) uint32 { - return 0x1e<<26 | (op&7)<<23 | (df&3)<<21 | uint32(s10&0x3FF)<<11 | (wd&31)<<6 | minor&0x3F -} - -func OP_VMI10(s10 int32, rs uint32, wd uint32, minor uint32, df uint32) uint32 { - return 0x1e<<26 | uint32(s10&0x3FF)<<16 | (rs&31)<<11 | (wd&31)<<6 | (minor&15)<<2 | df&3 -} - func (c *ctxt0) asmout(p *obj.Prog, o *Optab, out []uint32) { o1 := uint32(0) o2 := uint32(0) @@ -1670,19 +1649,6 @@ func (c *ctxt0) asmout(p *obj.Prog, o *Optab, out []uint32) { rel.Sym = p.From.Sym rel.Add = p.From.Offset rel.Type = objabi.R_ADDRMIPSTLS - - case 56: /* vmov{b,h,w,d} $scon, wr */ - - v := c.regoff(&p.From) - o1 = OP_VI10(110, c.twobitdf(p.As), v, uint32(p.To.Reg), 7) - - case 57: /* vld $soreg, wr */ - v := c.lsoffset(p.As, c.regoff(&p.From)) - o1 = OP_VMI10(v, uint32(p.From.Reg), uint32(p.To.Reg), 8, c.twobitdf(p.As)) - - case 58: /* vst wr, $soreg */ - v := c.lsoffset(p.As, c.regoff(&p.To)) - o1 = OP_VMI10(v, uint32(p.To.Reg), uint32(p.From.Reg), 9, c.twobitdf(p.As)) } out[0] = o1 @@ -2074,43 +2040,3 @@ func vshift(a obj.As) bool { } return false } - -// MSA Two-bit Data Format Field Encoding -func (c *ctxt0) twobitdf(a obj.As) uint32 { - switch a { - case AVMOVB: - return 0 - case AVMOVH: - return 1 - case AVMOVW: - return 2 - case AVMOVD: - return 3 - default: - c.ctxt.Diag("unsupported data format %v", a) - } - return 0 -} - -// MSA Load/Store offset have to be multiple of size of data format -func (c *ctxt0) lsoffset(a obj.As, o int32) int32 { - var mod int32 - switch a { - case AVMOVB: - mod = 1 - case AVMOVH: - mod = 2 - case AVMOVW: - mod = 4 - case AVMOVD: - mod = 8 - default: - c.ctxt.Diag("unsupported instruction:%v", a) - } - - if o%mod != 0 { - c.ctxt.Diag("invalid offset for %v: %d is not a multiple of %d", a, o, mod) - } - - return o / mod -} diff --git a/src/cmd/internal/obj/mips/list0.go b/src/cmd/internal/obj/mips/list0.go index f734e21ede1a8f..addf9f70d8a224 100644 --- a/src/cmd/internal/obj/mips/list0.go +++ b/src/cmd/internal/obj/mips/list0.go @@ -59,9 +59,6 @@ func rconv(r int) string { if REG_FCR0 <= r && r <= REG_FCR31 { return fmt.Sprintf("FCR%d", r-REG_FCR0) } - if REG_W0 <= r && r <= REG_W31 { - return fmt.Sprintf("W%d", r-REG_W0) - } if r == REG_HI { return "HI" } From 1a2d221d4bd86eeeb99853fe5a54646ee6ad3050 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Mon, 2 Oct 2023 23:48:25 +0200 Subject: [PATCH 04/45] Revert "runtime: improve MIPS64x memclr" This reverts commit 918d4d46cd17192a81a6aced57d09827560ad9f0. --- src/runtime/cpuflags.go | 2 -- src/runtime/memclr_mips64x.s | 55 ------------------------------------ 2 files changed, 57 deletions(-) diff --git a/src/runtime/cpuflags.go b/src/runtime/cpuflags.go index bbe93c5bea2d7e..1f562ea73eb6e3 100644 --- a/src/runtime/cpuflags.go +++ b/src/runtime/cpuflags.go @@ -17,8 +17,6 @@ const ( offsetX86HasRDTSCP = unsafe.Offsetof(cpu.X86.HasRDTSCP) offsetARMHasIDIVA = unsafe.Offsetof(cpu.ARM.HasIDIVA) - - offsetMIPS64XHasMSA = unsafe.Offsetof(cpu.MIPS64X.HasMSA) ) var ( diff --git a/src/runtime/memclr_mips64x.s b/src/runtime/memclr_mips64x.s index cf3a9c4ab4fb36..77cdcceb07f18b 100644 --- a/src/runtime/memclr_mips64x.s +++ b/src/runtime/memclr_mips64x.s @@ -4,7 +4,6 @@ //go:build mips64 || mips64le -#include "go_asm.h" #include "textflag.h" // See memclrNoHeapPointers Go doc for important implementation constraints. @@ -15,60 +14,6 @@ TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16 MOVV n+8(FP), R2 ADDV R1, R2, R4 - // if less than 16 bytes or no MSA, do words check - SGTU $16, R2, R3 - BNE R3, no_msa - MOVBU internal∕cpu·MIPS64X+const_offsetMIPS64XHasMSA(SB), R3 - BEQ R3, R0, no_msa - - VMOVB $0, W0 - - SGTU $128, R2, R3 - BEQ R3, msa_large - - AND $15, R2, R5 - XOR R2, R5, R6 - ADDVU R1, R6 - -msa_small: - VMOVB W0, (R1) - ADDVU $16, R1 - SGTU R6, R1, R3 - BNE R3, R0, msa_small - BEQ R5, R0, done - VMOVB W0, -16(R4) - JMP done - -msa_large: - AND $127, R2, R5 - XOR R2, R5, R6 - ADDVU R1, R6 - -msa_large_loop: - VMOVB W0, (R1) - VMOVB W0, 16(R1) - VMOVB W0, 32(R1) - VMOVB W0, 48(R1) - VMOVB W0, 64(R1) - VMOVB W0, 80(R1) - VMOVB W0, 96(R1) - VMOVB W0, 112(R1) - - ADDVU $128, R1 - SGTU R6, R1, R3 - BNE R3, R0, msa_large_loop - BEQ R5, R0, done - VMOVB W0, -128(R4) - VMOVB W0, -112(R4) - VMOVB W0, -96(R4) - VMOVB W0, -80(R4) - VMOVB W0, -64(R4) - VMOVB W0, -48(R4) - VMOVB W0, -32(R4) - VMOVB W0, -16(R4) - JMP done - -no_msa: // if less than 8 bytes, do one byte at a time SGTU $8, R2, R3 BNE R3, out From f82253ec237ca3d3edc149d575c9308338367c72 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Tue, 17 Oct 2023 16:10:41 +0200 Subject: [PATCH 05/45] Remove MUL instruction, unsupported in MIPS-III --- src/cmd/internal/obj/mips/asm0.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/cmd/internal/obj/mips/asm0.go b/src/cmd/internal/obj/mips/asm0.go index f65cce198cf819..b3302d91a76bf3 100644 --- a/src/cmd/internal/obj/mips/asm0.go +++ b/src/cmd/internal/obj/mips/asm0.go @@ -254,7 +254,6 @@ var optab = []Optab{ {AMOVV, C_REG, C_NONE, C_LO, 21, 4, 0, sys.MIPS64, 0}, {AMUL, C_REG, C_REG, C_NONE, 22, 4, 0, 0, 0}, - {AMUL, C_REG, C_REG, C_REG, 22, 4, 0, 0, 0}, {AMULV, C_REG, C_REG, C_NONE, 22, 4, 0, sys.MIPS64, 0}, {AADD, C_ADD0CON, C_REG, C_REG, 4, 4, 0, 0, 0}, @@ -1375,17 +1374,8 @@ func (c *ctxt0) asmout(p *obj.Prog, o *Optab, out []uint32) { } o1 = OP_RRR(a, REGZERO, p.From.Reg, REGZERO) - case 22: /* mul r1,r2 [r3]*/ - if p.To.Reg != obj.REG_NONE { - r := p.Reg - if r == obj.REG_NONE { - r = p.To.Reg - } - a := SP(3, 4) | 2 /* mul */ - o1 = OP_RRR(a, p.From.Reg, r, p.To.Reg) - } else { - o1 = OP_RRR(c.oprrr(p.As), p.From.Reg, p.Reg, REGZERO) - } + case 22: /* mul r1,r2 */ + o1 = OP_RRR(c.oprrr(p.As), p.From.Reg, p.Reg, REGZERO) case 23: /* add $lcon,r1,r2 ==> lu+or+add */ v := c.regoff(&p.From) From caf77a1a86216dfeca71c86d9a16369658960b10 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 27 Sep 2023 22:59:17 +0200 Subject: [PATCH 06/45] First steps for mips support --- src/cmd/dist/build.go | 1 + src/cmd/link/internal/mips64/obj.go | 11 + .../arch/r4000/systim/systim_mips64.go | 31 ++ src/embedded/mmio/asm_mips64.s | 54 +++ src/embedded/rtos/asm_mips64.s | 6 + src/embedded/rtos/irq_noos_mips64.go | 12 + src/internal/cpu/cpu_mips64x.go | 2 +- src/internal/cpu/cpu_noos_mips64.go | 9 + src/internal/cpu/r4000/creg/creg_mips64.go | 73 ++++ src/internal/cpu/r4000/creg/creg_mips64.s | 360 ++++++++++++++++++ src/internal/platform/zosarch.go | 2 + src/runtime/asm_mips64x.s | 4 + src/runtime/const_noos_n64.go | 36 ++ src/runtime/mksizeclasses_mcu.go | 2 +- src/runtime/mksizeclasses_mcu64.go | 2 +- src/runtime/os_noos_mips64.go | 12 + src/runtime/rt0_noos_mips64.s | 25 ++ src/runtime/sizeclasses_mcu64.go | 2 +- src/runtime/sys_noos_mips64.s | 151 ++++++++ src/runtime/tasker_noos_mips64.go | 80 ++++ src/runtime/tasker_noos_mips64.s | 4 + src/runtime/tls_mips64x.s | 2 +- src/runtime/tls_noos_mips64.s | 11 + 23 files changed, 887 insertions(+), 5 deletions(-) create mode 100644 src/embedded/arch/r4000/systim/systim_mips64.go create mode 100644 src/embedded/mmio/asm_mips64.s create mode 100644 src/embedded/rtos/asm_mips64.s create mode 100644 src/embedded/rtos/irq_noos_mips64.go create mode 100644 src/internal/cpu/cpu_noos_mips64.go create mode 100644 src/internal/cpu/r4000/creg/creg_mips64.go create mode 100644 src/internal/cpu/r4000/creg/creg_mips64.s create mode 100644 src/runtime/const_noos_n64.go create mode 100644 src/runtime/os_noos_mips64.go create mode 100644 src/runtime/rt0_noos_mips64.s create mode 100644 src/runtime/sys_noos_mips64.s create mode 100644 src/runtime/tasker_noos_mips64.go create mode 100644 src/runtime/tasker_noos_mips64.s create mode 100644 src/runtime/tls_noos_mips64.s diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index bb1d08e76d4e1c..edeffc3d91c385 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -1743,6 +1743,7 @@ var cgoEnabled = map[string]bool{ "windows/arm64": true, "noos/thumb": false, "noos/riscv64": false, + "noos/mips64": false, } // List of platforms that are marked as broken ports. diff --git a/src/cmd/link/internal/mips64/obj.go b/src/cmd/link/internal/mips64/obj.go index 193ad1f27110b8..e2d77d6f307c8e 100644 --- a/src/cmd/link/internal/mips64/obj.go +++ b/src/cmd/link/internal/mips64/obj.go @@ -105,6 +105,17 @@ func archinit(ctxt *ld.Link) { if *ld.FlagTextAddr == -1 { *ld.FlagTextAddr = ld.Rnd(0x10000, *ld.FlagRound) + int64(ld.HEADR) } + + case objabi.Hnoos: + *ld.FlagD = true + ld.Elfinit(ctxt) + ld.HEADR = ld.ELFRESERVE + if *ld.FlagTextAddr == -1 { + *ld.FlagTextAddr = 0 + } + if *ld.FlagRound == -1 { + *ld.FlagRound = 8 + } } dynSymCount = 0 diff --git a/src/embedded/arch/r4000/systim/systim_mips64.go b/src/embedded/arch/r4000/systim/systim_mips64.go new file mode 100644 index 00000000000000..7f6efc3cfb8db8 --- /dev/null +++ b/src/embedded/arch/r4000/systim/systim_mips64.go @@ -0,0 +1,31 @@ +package systim + +import ( + "embedded/rtos" + "internal/cpu/r4000/creg" + _ "unsafe" // for linkname +) + +var hz2ns int64 + +// Setup setups COUNT register to work as system timer. +func Setup(clkhz int64) { + hz2ns = 2 * 1e9 / clkhz + creg.COMPARE.Store(0xffffffff) + + rtos.SetSystemTimer(nanotime, setAlarm) +} + +//go:nosplit +func nanotime() int64 { + return int64(creg.COUNT.Load()) * hz2ns +} + +//go:nosplit +func setAlarm(ns int64) { + var compare uint32 + if ns >= 0 { + compare = uint32(ns / hz2ns) + } + creg.COMPARE.Store(compare) +} diff --git a/src/embedded/mmio/asm_mips64.s b/src/embedded/mmio/asm_mips64.s new file mode 100644 index 00000000000000..df679af2a735b6 --- /dev/null +++ b/src/embedded/mmio/asm_mips64.s @@ -0,0 +1,54 @@ +#include "textflag.h" + + +TEXT ·load64(SB),NOSPLIT,$0-16 + MOVV addr+0(FP), R4 + MOVV (R4), R5 + MOVV R5, ret+8(FP) + RET + +TEXT ·load32(SB),NOSPLIT,$0-12 + MOVV addr+0(FP), R4 + MOVWU (R4), R5 + MOVW R5, ret+8(FP) + RET + +TEXT ·load16(SB),NOSPLIT,$0-10 + MOVV addr+0(FP), R4 + MOVHU (R4), R5 + MOVH R5, ret+8(FP) + RET + +TEXT ·load8(SB),NOSPLIT,$0-9 + MOVV addr+0(FP), R4 + MOVBU (R4), R5 + MOVB R5, ret+8(FP) + RET + +TEXT ·store64(SB),NOSPLIT,$0-16 + MOVV addr+0(FP), R4 + MOVV v+8(FP), R5 + MOVV R5, (R4) + RET + +TEXT ·store32(SB),NOSPLIT,$0-12 + MOVV addr+0(FP), R4 + MOVWU v+8(FP), R5 + MOVW R5, (R4) + RET + +TEXT ·store16(SB),NOSPLIT,$0-10 + MOVV addr+0(FP), R4 + MOVHU v+8(FP), R5 + MOVH R5, (R4) + RET + +TEXT ·store8(SB),NOSPLIT,$0-9 + MOVV addr+0(FP), R4 + MOVBU v+8(FP), R5 + MOVB R5, (R4) + RET + +TEXT ·MB(SB),NOSPLIT,$0 + SYNC + RET diff --git a/src/embedded/rtos/asm_mips64.s b/src/embedded/rtos/asm_mips64.s new file mode 100644 index 00000000000000..31b19f9a1389b6 --- /dev/null +++ b/src/embedded/rtos/asm_mips64.s @@ -0,0 +1,6 @@ +#include "textflag.h" + +// func publicationBarrier() +TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 + SYNC + RET diff --git a/src/embedded/rtos/irq_noos_mips64.go b/src/embedded/rtos/irq_noos_mips64.go new file mode 100644 index 00000000000000..b6d27a9d338ee8 --- /dev/null +++ b/src/embedded/rtos/irq_noos_mips64.go @@ -0,0 +1,12 @@ +package rtos + +const ( + intPrioHighest = 0 + intPrioHigh = 0 + intPrioSysTimer = 0 + intPrioMid = 0 + intPrioSysCall = 0 + intPrioLow = 0 + intPrioLowest = 0 + intPrioCurrent = -1 +) diff --git a/src/internal/cpu/cpu_mips64x.go b/src/internal/cpu/cpu_mips64x.go index c452ffd8b30216..af1b2318a308d6 100644 --- a/src/internal/cpu/cpu_mips64x.go +++ b/src/internal/cpu/cpu_mips64x.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build mips64 || mips64le +//go:build (mips64 || mips64le) && !noos package cpu diff --git a/src/internal/cpu/cpu_noos_mips64.go b/src/internal/cpu/cpu_noos_mips64.go new file mode 100644 index 00000000000000..e998699fe04154 --- /dev/null +++ b/src/internal/cpu/cpu_noos_mips64.go @@ -0,0 +1,9 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const CacheLinePadSize = 32 + +func doinit() {} diff --git a/src/internal/cpu/r4000/creg/creg_mips64.go b/src/internal/cpu/r4000/creg/creg_mips64.go new file mode 100644 index 00000000000000..eae0004b79e4df --- /dev/null +++ b/src/internal/cpu/r4000/creg/creg_mips64.go @@ -0,0 +1,73 @@ +// Package creg provides access to the control registers. +// Page references refer to Vr4300 User' Manual (7th edition) +package creg + +type CP0Reg uint + +// System Control Processor (CP0) registers (p.46) +const ( + // memory management registers + INDEX CP0Reg = 0 + RANDOM CP0Reg = 1 + WIRED CP0Reg = 6 + PRID CP0Reg = 15 + CONFIG CP0Reg = 16 + LLADDR CP0Reg = 17 + ENTRY_LO_0 CP0Reg = 2 + ENTRY_LO_1 CP0Reg = 3 + ENTRY_HI CP0Reg = 10 + PAGE_MASK CP0Reg = 5 + CACHE_TAG_LO CP0Reg = 28 + CACHE_TAG_HI CP0Reg = 29 + + // exception processing registers + CONTEXT CP0Reg = 4 + BAD_VADDR CP0Reg = 8 + COUNT CP0Reg = 9 + COMPARE CP0Reg = 11 + STATUS CP0Reg = 12 + CAUSE CP0Reg = 13 + EPC CP0Reg = 14 + WATCH_LO CP0Reg = 18 + WATCH_HI CP0Reg = 19 + XCONTEXT CP0Reg = 20 + PARITY_ERR CP0Reg = 26 + CACHE_ERR CP0Reg = 27 + ERROR_EPC CP0Reg = 30 +) + +// Status register bits (p.166) +const ( + IE = 0x001 << 0 // interrupt enable + EXL = 0x001 << 1 // exception level + ERL = 0x001 << 2 // error level + KSU = 0x003 << 3 // kernel/supervisor/user mode + UX = 0x001 << 5 // user mode 64-bit addressing and operations + SX = 0x001 << 6 // supervisor mode 64-bit addressing and operations + KX = 0x001 << 7 // kernel mode 64-bit addressing + IM = 0x0ff << 8 // interrupt mask + DS = 0x1ff << 16 // diagnostic status + RE = 0x001 << 25 // reverse-endian + FR = 0x001 << 26 // floating-point registers + RP = 0x001 << 27 // reduced power + CU = 0x00f << 28 // coprocessor usability +) + +func (r CP0Reg) Load() uint32 { return mfc0(r) } +func (r CP0Reg) Store(val uint32) { mtc0(r, val) } + +func (r CP0Reg) LoadBits(mask uint32) uint32 { + return r.Load() & mask +} +func (r CP0Reg) StoreBits(mask uint32, val uint32) { + r.Store((r.Load() & ^mask) | (val & mask)) +} +func (r CP0Reg) SetBits(mask uint32) { + r.Store((r.Load() | mask)) +} +func (r CP0Reg) ClearBits(mask uint32) { + r.Store((r.Load() & ^mask)) +} + +func mtc0(creg CP0Reg, val uint32) +func mfc0(creg CP0Reg) uint32 diff --git a/src/internal/cpu/r4000/creg/creg_mips64.s b/src/internal/cpu/r4000/creg/creg_mips64.s new file mode 100644 index 00000000000000..e476af29d50161 --- /dev/null +++ b/src/internal/cpu/r4000/creg/creg_mips64.s @@ -0,0 +1,360 @@ +#include "textflag.h" + +TEXT ·mtc0(SB),NOSPLIT,$0-12 + MOVV creg+0(FP), R4 + MOVWU val+8(FP), R5 + MOVW $0, R6 + BEQ R6, R4, creg0 + MOVW $1, R6 + BEQ R6, R4, creg1 + MOVW $2, R6 + BEQ R6, R4, creg2 + MOVW $3, R6 + BEQ R6, R4, creg3 + MOVW $4, R6 + BEQ R6, R4, creg4 + MOVW $5, R6 + BEQ R6, R4, creg5 + MOVW $6, R6 + BEQ R6, R4, creg6 + MOVW $7, R6 + BEQ R6, R4, creg7 + MOVW $8, R6 + BEQ R6, R4, creg8 + MOVW $9, R6 + BEQ R6, R4, creg9 + MOVW $10, R6 + BEQ R6, R4, creg10 + MOVW $11, R6 + BEQ R6, R4, creg11 + MOVW $12, R6 + BEQ R6, R4, creg12 + MOVW $13, R6 + BEQ R6, R4, creg13 + MOVW $14, R6 + BEQ R6, R4, creg14 + MOVW $15, R6 + BEQ R6, R4, creg15 + MOVW $16, R6 + BEQ R6, R4, creg16 + MOVW $17, R6 + BEQ R6, R4, creg17 + MOVW $18, R6 + BEQ R6, R4, creg18 + MOVW $19, R6 + BEQ R6, R4, creg19 + MOVW $20, R6 + BEQ R6, R4, creg20 + MOVW $21, R6 + BEQ R6, R4, creg21 + MOVW $22, R6 + BEQ R6, R4, creg22 + MOVW $23, R6 + BEQ R6, R4, creg23 + MOVW $24, R6 + BEQ R6, R4, creg24 + MOVW $25, R6 + BEQ R6, R4, creg25 + MOVW $26, R6 + BEQ R6, R4, creg26 + MOVW $27, R6 + BEQ R6, R4, creg27 + MOVW $28, R6 + BEQ R6, R4, creg28 + MOVW $29, R6 + BEQ R6, R4, creg29 + MOVW $30, R6 + BEQ R6, R4, creg30 + MOVW $31, R6 + BEQ R6, R4, creg31 +creg0: + MOVW R5, M(0) + RET +creg1: + MOVW R5, M(1) + RET +creg2: + MOVW R5, M(2) + RET +creg3: + MOVW R5, M(3) + RET +creg4: + MOVW R5, M(4) + RET +creg5: + MOVW R5, M(5) + RET +creg6: + MOVW R5, M(6) + RET +creg7: + MOVW R5, M(7) + RET +creg8: + MOVW R5, M(8) + RET +creg9: + MOVW R5, M(9) + RET +creg10: + MOVW R5, M(10) + RET +creg11: + MOVW R5, M(11) + RET +creg12: + MOVW R5, M(12) + RET +creg13: + MOVW R5, M(13) + RET +creg14: + MOVW R5, M(14) + RET +creg15: + MOVW R5, M(15) + RET +creg16: + MOVW R5, M(16) + RET +creg17: + MOVW R5, M(17) + RET +creg18: + MOVW R5, M(18) + RET +creg19: + MOVW R5, M(19) + RET +creg20: + MOVW R5, M(20) + RET +creg21: + MOVW R5, M(21) + RET +creg22: + MOVW R5, M(22) + RET +creg23: + MOVW R5, M(23) + RET +creg24: + MOVW R5, M(24) + RET +creg25: + MOVW R5, M(25) + RET +creg26: + MOVW R5, M(26) + RET +creg27: + MOVW R5, M(27) + RET +creg28: + MOVW R5, M(28) + RET +creg29: + MOVW R5, M(29) + RET +creg30: + MOVW R5, M(30) + RET +creg31: + MOVW R5, M(31) + RET + +TEXT ·mfc0(SB),NOSPLIT,$0-12 + MOVV creg+0(FP), R4 + MOVW $0, R6 + BEQ R6, R4, creg0 + MOVW $1, R6 + BEQ R6, R4, creg1 + MOVW $2, R6 + BEQ R6, R4, creg2 + MOVW $3, R6 + BEQ R6, R4, creg3 + MOVW $4, R6 + BEQ R6, R4, creg4 + MOVW $5, R6 + BEQ R6, R4, creg5 + MOVW $6, R6 + BEQ R6, R4, creg6 + MOVW $7, R6 + BEQ R6, R4, creg7 + MOVW $8, R6 + BEQ R6, R4, creg8 + MOVW $9, R6 + BEQ R6, R4, creg9 + MOVW $10, R6 + BEQ R6, R4, creg10 + MOVW $11, R6 + BEQ R6, R4, creg11 + MOVW $12, R6 + BEQ R6, R4, creg12 + MOVW $13, R6 + BEQ R6, R4, creg13 + MOVW $14, R6 + BEQ R6, R4, creg14 + MOVW $15, R6 + BEQ R6, R4, creg15 + MOVW $16, R6 + BEQ R6, R4, creg16 + MOVW $17, R6 + BEQ R6, R4, creg17 + MOVW $18, R6 + BEQ R6, R4, creg18 + MOVW $19, R6 + BEQ R6, R4, creg19 + MOVW $20, R6 + BEQ R6, R4, creg20 + MOVW $21, R6 + BEQ R6, R4, creg21 + MOVW $22, R6 + BEQ R6, R4, creg22 + MOVW $23, R6 + BEQ R6, R4, creg23 + MOVW $24, R6 + BEQ R6, R4, creg24 + MOVW $25, R6 + BEQ R6, R4, creg25 + MOVW $26, R6 + BEQ R6, R4, creg26 + MOVW $27, R6 + BEQ R6, R4, creg27 + MOVW $28, R6 + BEQ R6, R4, creg28 + MOVW $29, R6 + BEQ R6, R4, creg29 + MOVW $30, R6 + BEQ R6, R4, creg30 + MOVW $31, R6 + BEQ R6, R4, creg31 +creg0: + MOVW M(0), R5 + MOVW R5, ret+8(FP) + RET +creg1: + MOVW M(1), R5 + MOVW R5, ret+8(FP) + RET +creg2: + MOVW M(2), R5 + MOVW R5, ret+8(FP) + RET +creg3: + MOVW M(3), R5 + MOVW R5, ret+8(FP) + RET +creg4: + MOVW M(4), R5 + MOVW R5, ret+8(FP) + RET +creg5: + MOVW M(5), R5 + MOVW R5, ret+8(FP) + RET +creg6: + MOVW M(6), R5 + MOVW R5, ret+8(FP) + RET +creg7: + MOVW M(7), R5 + MOVW R5, ret+8(FP) + RET +creg8: + MOVW M(8), R5 + MOVW R5, ret+8(FP) + RET +creg9: + MOVW M(9), R5 + MOVW R5, ret+8(FP) + RET +creg10: + MOVW M(10), R5 + MOVW R5, ret+8(FP) + RET +creg11: + MOVW M(11), R5 + MOVW R5, ret+8(FP) + RET +creg12: + MOVW M(12), R5 + MOVW R5, ret+8(FP) + RET +creg13: + MOVW M(13), R5 + MOVW R5, ret+8(FP) + RET +creg14: + MOVW M(14), R5 + MOVW R5, ret+8(FP) + RET +creg15: + MOVW M(15), R5 + MOVW R5, ret+8(FP) + RET +creg16: + MOVW M(16), R5 + MOVW R5, ret+8(FP) + RET +creg17: + MOVW M(17), R5 + MOVW R5, ret+8(FP) + RET +creg18: + MOVW M(18), R5 + MOVW R5, ret+8(FP) + RET +creg19: + MOVW M(19), R5 + MOVW R5, ret+8(FP) + RET +creg20: + MOVW M(20), R5 + MOVW R5, ret+8(FP) + RET +creg21: + MOVW M(21), R5 + MOVW R5, ret+8(FP) + RET +creg22: + MOVW M(22), R5 + MOVW R5, ret+8(FP) + RET +creg23: + MOVW M(23), R5 + MOVW R5, ret+8(FP) + RET +creg24: + MOVW M(24), R5 + MOVW R5, ret+8(FP) + RET +creg25: + MOVW M(25), R5 + MOVW R5, ret+8(FP) + RET +creg26: + MOVW M(26), R5 + MOVW R5, ret+8(FP) + RET +creg27: + MOVW M(27), R5 + MOVW R5, ret+8(FP) + RET +creg28: + MOVW M(28), R5 + MOVW R5, ret+8(FP) + RET +creg29: + MOVW M(29), R5 + MOVW R5, ret+8(FP) + RET +creg30: + MOVW M(30), R5 + MOVW R5, ret+8(FP) + RET +creg31: + MOVW M(31), R5 + MOVW R5, ret+8(FP) + RET diff --git a/src/internal/platform/zosarch.go b/src/internal/platform/zosarch.go index 9f9de5bf29fbd9..060e0567b1916a 100644 --- a/src/internal/platform/zosarch.go +++ b/src/internal/platform/zosarch.go @@ -46,6 +46,7 @@ var List = []OSArch{ {"netbsd", "arm64"}, {"noos", "riscv64"}, {"noos", "thumb"}, + {"noos", "mips64"}, {"openbsd", "386"}, {"openbsd", "amd64"}, {"openbsd", "arm"}, @@ -103,6 +104,7 @@ var distInfo = map[OSArch]osArchInfo{ {"netbsd", "arm64"}: {CgoSupported: true}, {"noos", "riscv64"}: {}, {"noos", "thumb"}: {}, + {"noos", "mips64"}: {}, {"openbsd", "386"}: {CgoSupported: true}, {"openbsd", "amd64"}: {CgoSupported: true}, {"openbsd", "arm"}: {CgoSupported: true}, diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 80cd87c4af335f..772df493a8f94f 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -11,6 +11,8 @@ #define REGCTXT R22 +#ifndef GOOS_noos + TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // R29 = stack; R4 = argc; R5 = argv @@ -74,6 +76,8 @@ nocgo: MOVV R0, 1(R0) RET +#endif + DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) GLOBL runtime·mainPC(SB),RODATA,$8 diff --git a/src/runtime/const_noos_n64.go b/src/runtime/const_noos_n64.go new file mode 100644 index 00000000000000..e9437e889938f7 --- /dev/null +++ b/src/runtime/const_noos_n64.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO: Do not use GOTARGET tags in runtime. Come up with something smarter +// based on some kind of memory description similar to the -M linker option. + +//go:build noos && n64 + +package runtime + +const ( + _OS = 0 + noos = true + noosScaleDown = 2 // must be power of 2 + noosStackCacheSize = 16 * 1024 + noosNumStackOrders = 3 + noosHeapAddrBits = 23 // enough for 8 MiB K210 SRAM + noosLogHeapArenaBytes = 17 // 128 KiB + noosArenaBaseOffset = 0xffffffff80000000 + noosMinPhysPageSize = 256 + noosSpanSetInitSpineCap = 64 + noosStackMin = 2048 + noosStackSystem = 0 + noosStackGuard = 928 + noosFinBlockSize = 2 * 1024 + noosSweepMinHeapDistance = 8 * 1024 + noosDefaultHeapMinimum = 64 * 1024 + noosMemoryLimitHeapGoalHeadroom = 1 << 18 + noosGCSweepBlockEntries = 1024 + noosGCSweepBufInitSpineCap = 64 + noosGCBitsChunkBytes = 16 * 1024 + noosSemTabSize = 113 + noosGOGC = 50 + noosTimeHistMaxBucketBits = 47 +) diff --git a/src/runtime/mksizeclasses_mcu.go b/src/runtime/mksizeclasses_mcu.go index 1c92eed35c6d75..b2b7d874f58373 100644 --- a/src/runtime/mksizeclasses_mcu.go +++ b/src/runtime/mksizeclasses_mcu.go @@ -52,7 +52,7 @@ func main() { fmt.Fprintln(&b, "// Code generated by mksizeclasses_mcu.go; DO NOT EDIT.") fmt.Fprintln(&b, "//go:generate go run mksizeclasses_mcu.go") fmt.Fprintln(&b) - fmt.Fprintln(&b, "//go:build noos && thumb") + fmt.Fprintln(&b, "//go:build noos && (thumb || mips)") fmt.Fprintln(&b) fmt.Fprintln(&b, "package runtime") classes := makeClasses() diff --git a/src/runtime/mksizeclasses_mcu64.go b/src/runtime/mksizeclasses_mcu64.go index a1828575c2c658..88dd1400ac2946 100644 --- a/src/runtime/mksizeclasses_mcu64.go +++ b/src/runtime/mksizeclasses_mcu64.go @@ -52,7 +52,7 @@ func main() { fmt.Fprintln(&b, "// Code generated by mksizeclasses_mcu64.go; DO NOT EDIT.") fmt.Fprintln(&b, "//go:generate go run mksizeclasses_mcu64.go") fmt.Fprintln(&b) - fmt.Fprintln(&b, "//go:build noos && riscv64") + fmt.Fprintln(&b, "//go:build noos && (riscv64 || mips64)") fmt.Fprintln(&b) fmt.Fprintln(&b, "package runtime") classes := makeClasses() diff --git a/src/runtime/os_noos_mips64.go b/src/runtime/os_noos_mips64.go new file mode 100644 index 00000000000000..6c7fd3c1c33c62 --- /dev/null +++ b/src/runtime/os_noos_mips64.go @@ -0,0 +1,12 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +//go:nosplit +func cputicks() int64 { + // Currently cputicks() is used in blocking profiler and to seed fastrand(). + // nanotime() is a poor approximation of CPU ticks that is enough for the profiler. + return nanotime() +} diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s new file mode 100644 index 00000000000000..563de9a01d5a28 --- /dev/null +++ b/src/runtime/rt0_noos_mips64.s @@ -0,0 +1,25 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "go_tls.h" +#include "funcdata.h" +#include "textflag.h" + + +TEXT _rt0_mips64_noos(SB),NOSPLIT|NOFRAME,$0 + // TODO implement + RET + + +#define PALLOC_MIN 20*1024 + +#define SCB_BASE 0xE000ED00 +#define SCB_VTOR 0x008 +#define SCB_CPACR 0x088 +#define SCB_FPCCR 0x234 + +TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 + // TODO implement + RET diff --git a/src/runtime/sizeclasses_mcu64.go b/src/runtime/sizeclasses_mcu64.go index 6809a55bf9420d..777f3b5f165fea 100644 --- a/src/runtime/sizeclasses_mcu64.go +++ b/src/runtime/sizeclasses_mcu64.go @@ -1,7 +1,7 @@ // Code generated by mksizeclasses_mcu64.go; DO NOT EDIT. //go:generate go run mksizeclasses_mcu64.go -//go:build noos && riscv64 +//go:build noos && (riscv64 || mips64) package runtime diff --git a/src/runtime/sys_noos_mips64.s b/src/runtime/sys_noos_mips64.s new file mode 100644 index 00000000000000..0e30655aea3ec7 --- /dev/null +++ b/src/runtime/sys_noos_mips64.s @@ -0,0 +1,151 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" +#include "syscall_noos.h" + +// if you add new syscall you must check sysMaxArgs in tasker_noos_*.s + +// syscalls allowed for low priority interrupt handlers +DATA runtime·syscalls+(SYS_nanotime*8)(SB)/8, $·sysnanotime(SB) +DATA runtime·syscalls+(SYS_irqctl*8)(SB)/8, $·sysirqctl(SB) +DATA runtime·syscalls+(SYS_setprivlevel*8)(SB)/8, $·syssetprivlevel(SB) +DATA runtime·syscalls+(SYS_write*8)(SB)/8, $·syswrite(SB) +DATA runtime·syscalls+(SYS_cachemaint*8)(SB)/8, $·syscachemaint(SB) + +// syscalls disallowed for low priority interrupt handlers +DATA runtime·syscalls+(SYS_setsystim1*8)(SB)/8, $·syssetsystim1(SB) +DATA runtime·syscalls+(SYS_setsyswriter1*8)(SB)/8, $·syssetsyswriter1(SB) +DATA runtime·syscalls+(SYS_newosproc*8)(SB)/8, $·sysnewosproc(SB) +DATA runtime·syscalls+(SYS_exitThread*8)(SB)/8, $·sysexitThread(SB) +DATA runtime·syscalls+(SYS_futexsleep*8)(SB)/8, $·sysfutexsleep(SB) +DATA runtime·syscalls+(SYS_futexwakeup*8)(SB)/8, $·sysfutexwakeup(SB) +DATA runtime·syscalls+(SYS_osyield*8)(SB)/8, $·curcpuSchedule(SB) +DATA runtime·syscalls+(SYS_nanosleep*8)(SB)/8, $·sysnanosleep(SB) +DATA runtime·syscalls+(SYS_reset*8)(SB)/8, $·sysreset(SB) + +GLOBL runtime·syscalls(SB), RODATA, $(SYS_NUM*8) + +// func nanotime() int64 +TEXT ·nanotime(SB),NOSPLIT|NOFRAME,$0-8 + MOVV $SYS_nanotime, R8 + MOVV $(0+8), R9 + MOVV $8, R10 + SYSCALL + RET + +// func irqctl(irq, ctl, ctxid int) (enabled, prio, errno int) +TEXT ·irqctl(SB),NOSPLIT|NOFRAME,$0-48 + MOVV $SYS_irqctl, R8 + MOVV $(24+8), R9 + MOVV $24, R10 + SYSCALL + RET + +// func setprivlevel(newlevel int) (oldlevel, errno int) +TEXT ·setprivlevel(SB),NOSPLIT|NOFRAME,$0-24 + MOVV $SYS_setprivlevel, R8 + MOVV $(8+8), R9 + MOVV $16, R10 + SYSCALL + RET + +// func write(fd uintptr, p unsafe.Pointer, n int32) int32 +TEXT ·write(SB),NOSPLIT|NOFRAME,$0-32 + MOVV $SYS_write, R8 + MOVV $(24+8), R9 + MOVV $8, R10 + SYSCALL + RET + +// func cachemaint(op int, p unsafe.Pointer, size int) +TEXT ·cachemaint(SB),NOSPLIT|NOFRAME,$0-24 + MOVV $SYS_cachemaint, R8 + MOVV $(24+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// func setsystim1() +TEXT ·setsystim1(SB),NOSPLIT|NOFRAME,$0-0 + MOVV $SYS_setsystim1, R8 + MOVV $(0+8), R9 + MOVV $0, R10 + SYSCALL + RET + + +// func setsyswriter1() +TEXT ·setsyswriter1(SB),NOSPLIT|NOFRAME,$0-0 + MOVV $SYS_setsyswriter1, R8 + MOVV $(0+8), R9 + MOVV $0, R10 + SYSCALL + RET + + +// func newosproc(mp *m) +TEXT ·newosproc(SB),NOSPLIT|NOFRAME,$0-8 + MOVV $SYS_newosproc, R8 + MOVV $(8+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// func exitThread(wait *atomic.Uint32) +TEXT ·exitThread(SB),NOSPLIT|NOFRAME,$0-8 + MOVV $SYS_exitThread, R8 + MOVV $(8+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// func futexsleep(addr *uint32, val uint32, ns int64) +TEXT ·futexsleep(SB),NOSPLIT|NOFRAME,$0-24 + MOVV $SYS_futexsleep, R8 + MOVV $(24+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// func futexwakeup(addr *uint32, cnt uint32) +TEXT ·futexwakeup(SB),NOSPLIT|NOFRAME,$0-16 + MOVV $SYS_futexwakeup, R8 + MOVV $(16+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// func osyield() +TEXT ·osyield(SB),NOSPLIT|NOFRAME,$0-0 + MOVV $SYS_osyield, R8 + MOVV $(0+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// func nanosleep(ns int64) +TEXT ·nanosleep(SB),NOSPLIT|NOFRAME,$0-8 + MOVV $SYS_nanosleep, R8 + MOVV $(8+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// func reset(level int, addr unsafe.Pointer) bool +TEXT ·reset(SB),NOSPLIT|NOFRAME,$0-16 + MOVV $SYS_reset, R8 + MOVV $(16+8), R9 + MOVV $0, R10 + SYSCALL + RET + +// unsupported syscalls + +// func exit(r int32) +TEXT ·exit(SB),NOSPLIT|NOFRAME,$0-8 + BREAK + JMP -1(PC) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go new file mode 100644 index 00000000000000..1de01ac257f8eb --- /dev/null +++ b/src/runtime/tasker_noos_mips64.go @@ -0,0 +1,80 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +type mOS [1]uint32 + +// This functions is called to put the CPU to sleep. It is allowed it does +// nothing. +// +//go:nosplit +func curcpuSleep() {} + +// This function is used to inform the another CPU that there is a new thread +// added to its runnable queue. It should wake up the sleeping CPU or preempt +// the currently running thread to run the scheduler. The thread preemption can +// be set as delayed to allow a running thread to run for a minimum period of +// time. +// +//go:nosplit +func (cpu *cpuctx) newwork() { + // for now only single CPU is supported +} + +//go:nosplit +func curcpuWakeup() { + // TODO implement +} + +// This function is called to save the remaining context, not saved at syscall +// entry (eg. it can save FPU state). +// +//go:nosplit +func curcpuSavectxSched() { + // TODO implement +} + +// This function is called to save the remaining context, not saved at syscall +// entry (eg. it can save FPU state). +// +//go:nosplit +func curcpuSavectxCall() { + // TODO implement +} + +//go:nosplit +func cpuid() int { + // for now only single CPU is supported + return 0 +} + +// This function is called to create the inintial state of the new thread and +// save it in provided m. +// +//go:nosplit +func archnewm(m *m) { + // TODO implement +} + +// Run scheduler immediately or at syscall exit. It's called only just before +// syscall exit. +// +// The actual context switch is performed by architecture specific code at +// curcpuRunScheduler exit. It should check the cpuctx.newexe variable and if +// true switch the context to the new thread specified in cpuctx.exe. +// +// Tasker code does not use FPU so the architecture specific context switch +// code can avoid saving/restoring FPU context if not need. +// +//go:nosplit +func curcpuSchedule() { + // TODO implement +} + +//go:nowritebarrierrec +//go:nosplit +func defaultWrite(fd int, p []byte) int { + return 0 +} diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s new file mode 100644 index 00000000000000..0d847a5d98686b --- /dev/null +++ b/src/runtime/tasker_noos_mips64.s @@ -0,0 +1,4 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + diff --git a/src/runtime/tls_mips64x.s b/src/runtime/tls_mips64x.s index ec2748e5b22f03..1e7aac99c9f207 100644 --- a/src/runtime/tls_mips64x.s +++ b/src/runtime/tls_mips64x.s @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build mips64 || mips64le +//go:build (mips64 || mips64le) && !noos #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/tls_noos_mips64.s b/src/runtime/tls_noos_mips64.s new file mode 100644 index 00000000000000..a64e501ca9a421 --- /dev/null +++ b/src/runtime/tls_noos_mips64.s @@ -0,0 +1,11 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT runtime·save_g(SB),NOSPLIT|NOFRAME,$0-0 + RET + +TEXT runtime·load_g(SB),NOSPLIT|NOFRAME,$0-0 + RET From 8d2c24373fb124158efaa00cd2effd7aa37262cb Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sat, 30 Sep 2023 23:36:56 +0200 Subject: [PATCH 07/45] move entry function to beginning of text section --- src/cmd/link/internal/mips64/asm.go | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/cmd/link/internal/mips64/asm.go b/src/cmd/link/internal/mips64/asm.go index e82d9861841109..38c486b6bb70d3 100644 --- a/src/cmd/link/internal/mips64/asm.go +++ b/src/cmd/link/internal/mips64/asm.go @@ -61,7 +61,39 @@ var ( gotSymIndex uint64 ) +func lookupFuncSym(ldr *loader.Loader, name string) loader.Sym { + if s := ldr.Lookup(name, sym.SymVerABI0); s != 0 && ldr.SymType(s) == sym.STEXT { + return s + } + if s := ldr.Lookup(name, sym.SymVerABIInternal); s != 0 && ldr.SymType(s) == sym.STEXT { + return s + } + return 0 +} + +func gentext_noos(ctxt *ld.Link, ldr *loader.Loader) { + // TODO support interrupt vector table + + entry := lookupFuncSym(ldr, *ld.FlagEntrySymbol) + if entry == 0 { + ld.Errorf(nil, "cannot find entry function: %s", *ld.FlagEntrySymbol) + } + for i, s := range ctxt.Textp { + if s == entry { + copy(ctxt.Textp[1:], ctxt.Textp[:i]) + ctxt.Textp[0] = s + return + } + } + ldr.Errorf(entry, "cannot find symbol in ctxt.Textp") +} + func gentext(ctxt *ld.Link, ldr *loader.Loader) { + if ctxt.HeadType == objabi.Hnoos { + gentext_noos(ctxt, ldr) + return + } + if *ld.FlagD || ctxt.Target.IsExternal() { return } From c9f68a39409a69d58b5e52c3aecb469b877a52af Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sat, 7 Oct 2023 21:33:16 +0200 Subject: [PATCH 08/45] Implement _rt0_mips64_noos and rt0_go --- src/runtime/asm_mips64.h | 169 ++++++++++++++++++++ src/runtime/rt0_noos_mips64.s | 250 ++++++++++++++++++++++++++++-- src/runtime/tasker_noos_mips64.go | 29 +++- src/runtime/tasker_noos_mips64.s | 20 +++ 4 files changed, 458 insertions(+), 10 deletions(-) create mode 100644 src/runtime/asm_mips64.h diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h new file mode 100644 index 00000000000000..ef1170d40a8a37 --- /dev/null +++ b/src/runtime/asm_mips64.h @@ -0,0 +1,169 @@ +/* + * standard MIPS register names. + * + * Copyright (c) 1995 Cygnus Support + * + * The authors hereby grant permission to use, copy, modify, distribute, + * and license this software and its documentation for any purpose, provided + * that existing copyright notices are retained in all copies and that this + * notice is included verbatim in any distributions. No written agreement, + * license, or royalty fee is required for any of the authorized uses. + * Modifications to this software may be copyrighted by their authors + * and need not follow the licensing terms described here, provided that + * the new terms are clearly indicated on the first page of each file where + * they apply. + */ + +/* Standard MIPS register names: */ +#define zero R0 +#define z0 R0 +#define at R1 +#define v0 R2 +#define v1 R3 +#define a0 R4 +#define a1 R5 +#define a2 R6 +#define a3 R7 +#define t0 R8 +#define t1 R9 +#define t2 R10 +#define t3 R11 +#define t4 R12 +#define t5 R13 +#define t6 R14 +#define t7 R15 +#define s0 R16 +#define s1 R17 +#define s2 R18 +#define s3 R19 +#define s4 R20 +#define s5 R21 +#define s6 R22 +#define s7 R23 +#define t8 R24 +#define t9 R25 +#define k0 R26 /* kernel private register 0 */ +#define k1 R27 /* kernel private register 1 */ +#define gp R28 /* global data pointer */ +#define sp R29 /* stack-pointer */ +#define fp R30 /* frame-pointer */ +#define ra R31 /* return address */ + +#define fp0 F0 +#define fp1 F1 + +/* Useful memory constants: */ +#define K0BASE 0x80000000 +#ifndef __mips64 +#define K1BASE 0xA0000000 +#define K0BASE_ADDR ((char *)K0BASE) +#define K1BASE_ADDR ((char *)K1BASE) +#else +#define K1BASE 0xFFFFFFFFA0000000LL +#define K0BASE_ADDR ((char *)0xFFFFFFFF80000000LL) +#define K1BASE_ADDR ((char *)K1BASE) +#endif + +#define PHYS_TO_K1(a) ((unsigned)(a) | K1BASE) + +/* Standard Co-Processor 0 registers */ +#define C0_COUNT 9 /* Timer Count Register */ +#define C0_COMPARE 11 /* Timer Compare Register */ +#define C0_SR 12 /* Status Register */ +#define C0_CAUSE 13 /* last exception description */ +#define C0_EPC 14 /* Exception error address */ +#define C0_PRID 15 /* Processor Revision ID */ +#define C0_CONFIG 16 /* CPU configuration */ +#define C0_WATCHLO 18 /* Watchpoint */ + +/* Standard Processor Revision ID Register field offsets */ +#define PR_IMP 8 + +/* Standard Config Register field offsets */ +#define CR_DB 4 +#define CR_IB 5 +#define CR_DC 6 /* NOTE v4121 semantics != 43,5xxx semantics */ +#define CR_IC 9 /* NOTE v4121 semantics != 43,5xxx semantics */ +#define CR_SC 17 +#define CR_SS 20 +#define CR_SB 22 + + +/* Standard Status Register bitmasks: */ +#define SR_CU1 0x20000000 /* Mark CP1 as usable */ +#define SR_FR 0x04000000 /* Enable MIPS III FP registers */ +#define SR_BEV 0x00400000 /* Controls location of exception vectors */ +#define SR_PE 0x00100000 /* Mark soft reset (clear parity error) */ + +#define SR_KX 0x00000080 /* Kernel extended addressing enabled */ +#define SR_SX 0x00000040 /* Supervisor extended addressing enabled */ +#define SR_UX 0x00000020 /* User extended addressing enabled */ + +#define SR_ERL 0x00000004 /* Error level */ +#define SR_EXL 0x00000002 /* Exception level */ +#define SR_IE 0x00000001 /* Interrupts enabled */ + +/* Standard Cause Register bitmasks: */ +#define CAUSE_EXC_MASK (0x1F << 2) +#define CAUSE_EXC_INTERRUPT (0 << 2) +#define CAUSE_EXC_SYSCALL (8 << 2) +#define CAUSE_EXC_BREAKPOINT (9 << 2) +#define CAUSE_EXC_COPROCESSOR (11 << 2) + +#define CAUSE_IP_MASK (0xFF << 8) +#define CAUSE_IP7 (0x80 << 8) + +/* Standard (R4000) cache operations. Taken from "MIPS R4000 + Microprocessor User's Manual" 2nd edition: */ + +#define CACHE_I (0) /* primary instruction */ +#define CACHE_D (1) /* primary data */ +#define CACHE_SI (2) /* secondary instruction */ +#define CACHE_SD (3) /* secondary data (or combined instruction/data) */ + +#define INDEX_INVALIDATE (0) /* also encodes WRITEBACK if CACHE_D or CACHE_SD */ +#define INDEX_LOAD_TAG (1) +#define INDEX_STORE_TAG (2) +#define CREATE_DIRTY_EXCLUSIVE (3) /* CACHE_D and CACHE_SD only */ +#define HIT_INVALIDATE (4) +#define CACHE_FILL (5) /* CACHE_I only */ +#define HIT_WRITEBACK_INVALIDATE (5) /* CACHE_D and CACHE_SD only */ +#define HIT_WRITEBACK (6) /* CACHE_I, CACHE_D and CACHE_SD only */ +#define HIT_SET_VIRTUAL (7) /* CACHE_SI and CACHE_SD only */ + +#define BUILD_CACHE_OP(o,c) (((o) << 2) | (c)) + +/* Individual cache operations: */ +#define INDEX_INVALIDATE_I BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_I) +#define INDEX_WRITEBACK_INVALIDATE_D BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_D) +#define INDEX_INVALIDATE_SI BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_SI) +#define INDEX_WRITEBACK_INVALIDATE_SD BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_SD) + +#define INDEX_LOAD_TAG_I BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_I) +#define INDEX_LOAD_TAG_D BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_D) +#define INDEX_LOAD_TAG_SI BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_SI) +#define INDEX_LOAD_TAG_SD BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_SD) + +#define INDEX_STORE_TAG_I BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_I) +#define INDEX_STORE_TAG_D BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_D) +#define INDEX_STORE_TAG_SI BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_SI) +#define INDEX_STORE_TAG_SD BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_SD) + +#define CREATE_DIRTY_EXCLUSIVE_D BUILD_CACHE_OP(CREATE_DIRTY_EXCLUSIVE,CACHE_D) +#define CREATE_DIRTY_EXCLUSIVE_SD BUILD_CACHE_OP(CREATE_DIRTY_EXCLUSIVE,CACHE_SD) + +#define HIT_INVALIDATE_I BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_I) +#define HIT_INVALIDATE_D BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_D) +#define HIT_INVALIDATE_SI BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_SI) +#define HIT_INVALIDATE_SD BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_SD) + +#define CACHE_FILL_I BUILD_CACHE_OP(CACHE_FILL,CACHE_I) +#define HIT_WRITEBACK_INVALIDATE_D BUILD_CACHE_OP(HIT_WRITEBACK_INVALIDATE,CACHE_D) +#define HIT_WRITEBACK_INVALIDATE_SD BUILD_CACHE_OP(HIT_WRITEBACK_INVALIDATE,CACHE_SD) + +#define HIT_WRITEBACK_I BUILD_CACHE_OP(HIT_WRITEBACK,CACHE_I) +#define HIT_WRITEBACK_D BUILD_CACHE_OP(HIT_WRITEBACK,CACHE_D) +#define HIT_WRITEBACK_SD BUILD_CACHE_OP(HIT_WRITEBACK,CACHE_SD) + +#define HIT_SET_VIRTUAL_SI BUILD_CACHE_OP(HIT_SET_VIRTUAL,CACHE_SI) +#define HIT_SET_VIRTUAL_SD BUILD_CACHE_OP(HIT_SET_VIRTUAL,CACHE_SD) diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 563de9a01d5a28..73f6f91d794c86 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -6,20 +6,252 @@ #include "go_tls.h" #include "funcdata.h" #include "textflag.h" +#include "asm_mips64.h" +#define DEFAULT_C0_SR SR_CU1|SR_PE|SR_FR TEXT _rt0_mips64_noos(SB),NOSPLIT|NOFRAME,$0 - // TODO implement - RET + // Watchpoints have been proven to persist across resets and even with + // the console being off. Zero it as early as possible, to avoid it + // triggering during boot. This should really be done at the start IPL3. + MOVW z0, M(C0_WATCHLO) + // Check whether we are running on iQue or N64. Use the MI version + // register which has LSB set to 0xB0 on iQue. We assume 0xBn was meant + // for BBPlayer. Notice that we want this test to be hard for emulators + // to pass by mistake, so checking for a specific value while reading + // seems solid enough. + MOVW (0x80000318), t0 // memory size + MOVW (0xA4300004), t1 + MOVW $0xB0, t2 + AND $0xF0, t1 + MOVW $0, t7 // t7=0 -> vanilla N64 + BNE t1, t2, set_sp -#define PALLOC_MIN 20*1024 + // In iQue player, memory allocated to game can be configured and it + // appears in 0x80000318. On the other hand, the top 8 MiB of RDRAM is + // reserved to savegames. So avoid putting the stack there, capping the + // size to 0x7C0000. See also get_memory_size. + MOVW $1, t7 // t7=1 -> iQue player + MOVW 0x800000, t1 + SGT t2, t1, t0 + MOVW 0x0, t1 + BNE t2, t1, set_sp + MOVW 0x7C0000, t0 + +set_sp: + MOVW $0x7FFFFFF0, t1 + ADDU t0,t1,sp // init stack pointer + MOVV $0, RSB // init data pointer + MOVW $8, v0 + MOVW v0, (0xbfc007fc) // magic N64 hardware init + + // a bit from libgloss so we start at a known state + MOVW $DEFAULT_C0_SR, v0 + MOVW v0, M(C0_SR) + MOVW $0, M(C0_CAUSE) + + // Check if PI DMA transfer is required, knowing that IPL3 loads 1 MiB + // of ROM to RAM, and __libdragon_text_start is located right after the + // ROM header where this 1 MiB starts. + MOVW $_rt0_mips64_noos(SB), a0 + MOVW $runtime·edata(SB), a1 + MOVW $0x100000, t0 // stock IPL3 load size (1 MiB) + SUBU a0, a1, a2 // calculate data size + SUB t0, a2, a2 // calculate remaining data size + BLEZ a2, skip_dma // skip PI DMA if data is already loaded + + // Copy code and data via DMA + MOVW $0x10001000, a1 // address in rom + ADDU t0, a0, a0 // skip over loaded data + ADDU t0, a1, a1 + + // Start PI DMA transfer + MOVW $0xA4600000, t0 + MOVW a0, 0x00(t0) // PI_DRAM_ADDR + MOVW a1, 0x04(t0) // PI_CART_ADDR + ADD $-1, a2 + MOVW a2, 0x0C(t0) // PI_WR_LEN + +skip_dma: + // fill .bss with 0s + SUBU $16, sp + MOVW $runtime·bss(SB), a0 + OR $0x20000000, a0 // convert address to KSEG1 (uncached) + MOVW $runtime·ebss(SB), a1 + OR $0x20000000, a1 + SUB a0, a1 + MOVV a0, 8(sp) + MOVV a1, 16(sp) + JAL runtime·memclrNoHeapPointers(SB) // clear BSS + + // fill .noptrbss with 0s + MOVW $runtime·noptrbss(SB), a0 + OR $0x20000000, a0 // convert address to KSEG1 (uncached) + MOVW $runtime·enoptrbss(SB), a1 + OR $0x20000000, a1 + SUB a0, a1 + MOVV a0, 8(sp) + MOVV a1, 16(sp) + JAL runtime·memclrNoHeapPointers(SB) // clear noptrBSS + + // clear unallocated memory + MOVW $runtime·end(SB), a0 + OR $0x20000000, a0 // convert address to KSEG1 (uncached) + MOVW $runtime·ramend(SB), a1 + OR $0x20000000, a1 + SUB a0, a1 + MOVV a0, 8(sp) + MOVV a1, 16(sp) + JAL runtime·memclrNoHeapPointers(SB) // clear unallocated memory + ADDU $16, sp + + // Wait for DMA transfer to be finished + MOVW $0xA4600000, t0 -#define SCB_BASE 0xE000ED00 -#define SCB_VTOR 0x008 -#define SCB_CPACR 0x088 -#define SCB_FPCCR 0x234 +wait_dma_end: + MOVW 0x10(t0), t1 // PI_STATUS + AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ t1, wait_dma_end + + // Store the bbplayer flag now that BSS has been cleared + MOVW t7, runtime·bbplayer(SB) + + // load interrupt vector + MOVW $·intvector(SB), t0 + MOVW $0xa0000000, t1 + MOVW $4, t2 + +loadintvectorloop: + MOVW (t0), t3 + MOVW t3, 0(t1) + MOVW t3, 0x80(t1) + MOVW t3, 0x100(t1) + MOVW t3, 0x180(t1) + // sync - BREAK is overloaded CACHE opcode + BREAK R16, 0(t1) + BREAK R16, 0x80(t1) + BREAK R16, 0x100(t1) + BREAK R16, 0x180(t1) + ADD $4, t0 + ADD $4, t1 + ADDU $-1, t2 + BGTZ t2,loadintvectorloop + + JMP runtime·rt0_go(SB) + + +#define PALLOC_MIN 20*1024 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 - // TODO implement - RET + // setup main stack in cpu0.gh + MOVV $runtime·cpu0(SB), t0 // gh is the first field of the cpuctx struct + MOVV $runtime·ramend(SB), sp // main stack starts at the end of memory + SUB $16, sp + MOVV sp, (g_stack+stack_hi)(t0) + SUB $4096, sp, t1 + MOVV t1, (g_stack+stack_lo)(t0) + ADD $const_stackGuard, t1 + MOVV t1, g_stackguard0(t0) + MOVV t1, g_stackguard1(t0) + + // set up m0 (bootstrap thread), temporarily use cpu0.gh as g + MOVV $runtime·m0(SB), t1 + MOVV t0, m_g0(t1) // m0.g0 = cpu0.gh + MOVV t1, g_m(t0) // cpu0.gh.m = m0 + + MOVV t0, g // set g to gh + + JAL runtime·check(SB) + JAL runtime·osinit(SB) + + // initialize noosMem + + MOVV $runtime·end(SB), t0 + MOVV $runtime·ramend(SB), t1 + SUB $4096, t1 + SUB t0, t1, t5 // size of available memory (DMA capable) + + // estimate the space need for non-heap allocations + SRL $(const__PageShift+1), t5, t4 + MOVV $mspan__size, t2 + MUL t2, t4 + MOVV LO, t4 + ADD $PALLOC_MIN, t4 + + MOVV $runtime·nodmastart(SB), t2 + MOVV $runtime·nodmaend(SB), t3 + SUB t2, t3, t7 // size of non-DMA memory + ADD t5, t7, t6 // size of the whole free memory + + // we prefer the non-DMA memory for non-heap objects to preserve as much as + // possible of the DMA capable memory for heap allocations + SUB t7, t4 + + // reduce the arena by the remain of the non-heap space that did not fit in + // the non-DMA memory, properly align the arena + BLTZ t4, 2(PC) + SUB t4, t5 + AND $~(const_heapArenaBytes-1), t5 + SUB t5, t1 + MOVV t1, t4 + + // save {free.start,free.end,nodma.start,nodma.end,arenaStart,arenaSize,size} + MOVV $runtime·noosMem(SB), t7 + MOVV t0, 0(t7) + MOVV t1, 8(t7) + MOVV t2, 16(t7) + MOVV t3, 24(t7) + MOVV t4, 32(t7) + MOVV t5, 40(t7) + MOVV t6, 48(t7) + + // initialize noos tasker and Go scheduler + JAL runtime·taskerinit(SB) + JAL runtime·schedinit(SB) + + // allocate g0 for m0 and leave gh + SUB $16, sp + MOVW $(2*const_stackMin), a0 + MOVW a0, 8(sp) + JAL runtime·malg(SB) + MOVV 16(sp), t0 // newg in t0 + ADD $16, sp + + // stackguard check during newproc requires valid stackguard1 but + // malg sets it to 0xFFFFFFFF (mstart fixes this but is called later) + MOVV g_stackguard0(t0), t1 + MOVV t1, g_stackguard1(t0) + + MOVV $runtime·m0(SB), t1 + MOVV t0, m_g0(t1) // m0.g0 = newg + MOVV t1, g_m(t0) // newg.m = m0 + + // newg stack pointer to sp + MOVV (g_stack+stack_hi)(t0), sp + + // newg to g + MOVV g, t2 + MOVV t0, g + + // fix cpu0.gh, cpu0.mh + ADD $cpuctx_mh, t2, t1 // t2 points to cpu0 (and to cpu0.gh at the same time) + MOVV t2, m_g0(t1) // cpu0.mh.g0 = cpu0.gh + MOVV t2, m_gsignal(t1) // cpu0.mh.gsignal = cpu0.gh (to easily check for handler mode) + MOVV t1, g_m(t2) // cpu0.gh.m = cpu0.mh + + // TODO switch to the user mode? + + // create a new goroutine to start program + SUB $16, sp + MOVV $runtime·mainPC(SB), t0 + MOVV t0, 8(sp) // arg 1: fn + MOVV $0, t0 + MOVV t0, 0(sp) // dummy LR + JAL runtime·newproc(SB) + ADD $16, sp // pop args and LR + + // start this M + JAL runtime·mstart(SB) + + UNDEF diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index 1de01ac257f8eb..440d219a0b6ebc 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -4,8 +4,30 @@ package runtime +import ( + "unsafe" +) + type mOS [1]uint32 +var bbplayer bool // TODO move to n64 board support package + +var ( + cpu0 cpuctx + pcpu0 = &cpu0 +) + +//go:nowritebarrierrec +//go:nosplit +func taskerinit() { + *(*uintptr)(unsafe.Pointer(&cpu0.t)) = uintptr(unsafe.Pointer(&thetasker)) + cpu0.exe.set(getg().m) + allcpu := (*slice)(unsafe.Pointer(&thetasker.allcpu)) + *(*uintptr)(unsafe.Pointer(&allcpu.array)) = uintptr(unsafe.Pointer(&pcpu0)) + allcpu.len = 1 + allcpu.cap = 1 +} + // This functions is called to put the CPU to sleep. It is allowed it does // nothing. // @@ -70,11 +92,16 @@ func archnewm(m *m) { // //go:nosplit func curcpuSchedule() { + curcpu().schedule = true // TODO implement } //go:nowritebarrierrec //go:nosplit func defaultWrite(fd int, p []byte) int { - return 0 + return len(p) } + +// syscalls not used by runtime +func syssetprivlevel(newlevel int) (oldlevel, errno int) { return } // TODO +func sysirqctl(irq, ctl, ctxid int) (enabled, prio, errno int) { return } // TODO diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 0d847a5d98686b..54f7f616dc662e 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -2,3 +2,23 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" +#include "syscall_noos.h" + +#include "asm_mips64.h" + +TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 + JMP ·inthandler(SB) + +TEXT runtime·inthandler(SB),NOSPLIT|NOFRAME,$0 + RET // TODO implement + +// func sysreset(level int, addr unsafe.Pointer) bool +TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 + NOP // TODO + +// func syscachemaint(op int, p unsafe.Pointer, size int) +TEXT ·syscachemaint(SB),NOSPLIT,$0-12 + NOP // TODO From 6d41f48fcff392d007bfab956a860e4e7f7c3e30 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Fri, 13 Oct 2023 09:35:12 +0200 Subject: [PATCH 09/45] First draft for interrupt handler Saving, restoring context and switching to the ISR goroutine (cpu0.gh) seems to work so far. Only syscalls and interrupts are handled so far. See TODOs for next steps. --- src/runtime/asm_mips64.h | 6 ++ src/runtime/tasker_noos_mips64.s | 135 ++++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index ef1170d40a8a37..d3857972573ab2 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -113,6 +113,12 @@ #define CAUSE_IP_MASK (0xFF << 8) #define CAUSE_IP7 (0x80 << 8) +// Exception Context +#define _LR (0*8) +#define _mcause (1*8) +#define _mepc (2*8) +#define excCtxSize (3*8) + /* Standard (R4000) cache operations. Taken from "MIPS R4000 Microprocessor User's Manual" 2nd edition: */ diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 54f7f616dc662e..dccf4f6c2ec81f 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -9,11 +9,144 @@ #include "asm_mips64.h" +#define sysMaxArgs (48+8) + TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 JMP ·inthandler(SB) +// main exception handler +// +// interrupt exceptions are disabled by EXL=1 but other exceptions can +// still occur. TEXT runtime·inthandler(SB),NOSPLIT|NOFRAME,$0 - RET // TODO implement + // R26 and R27 are free, see runtime/asm_mips64x.s + // switch to special ISR goroutine + MOVV $·cpu0(SB), R26 + BNE R26, g, fromThread + MOVV $1, R27 // fromHandler flag + JMP fromHandler + +fromThread: + MOVV $0, R27 + MOVV R29, (g_sched+gobuf_sp)(R26) + MOVV g, (g_sched+gobuf_g)(R26) + MOVV (g_stack+stack_hi)(R26), R29 + MOVV R26, g + +fromHandler: + // save exception context + SUB $excCtxSize, R29 + MOVV R31, _LR(R29) + MOVV M(C0_CAUSE), R26 + MOVV R26, _mcause(R29) + MOVV M(C0_EPC), R26 + OR R27, R26 // encode fromHandler flag in EPC + MOVV R26, _mepc(R29) + + // TODO reenable interrupts? they might clobber R26, R27 + + // branch depending on exception cause + MOVV _mcause(R29), R26 + AND $CAUSE_EXC_MASK, R26 + + MOVV $CAUSE_EXC_SYSCALL, R27 + BEQ R26, R27, syscall + + MOVV $CAUSE_EXC_INTERRUPT, R27 + BEQ R26, R27, interrupt + + JMP fatal + +syscall: + // R8 - syscall number, + // R9 - argument data size on the stack (+8 for frame-pointer), + // R10 - return data size on the stack. + // + // Now that we know we are in a syscall, we are free to use any + // registers. The caller should expect them to be clobbered after a + // function call. + + // determine caller stack + MOVV _mepc(R29), R26 + AND $1, R26, R27 // fromHandler flag + BNE R27, R0, currentStack + + MOVV (g_sched+gobuf_sp)(g), R1 // duffcopy src thread + JMP continue + +currentStack: + ADD $excCtxSize, R29, R1 // duffcopy src handler + +continue: + // 3 extra registers to preserve src, dst and size of result + SUB $sysMaxArgs+3*8, R29 + + // copy arguments from the caller's stack + MOVV $·duffcopy+2048(SB), R26 + SLL $1, R9 + SUB R9, R26 + MOVV R29, R2 // duffcopy dst + JAL (R26) + + // save data needed to copy the return values back to the caller's stack + MOVV R1, (sysMaxArgs+0*8)(R29) + MOVV R2, (sysMaxArgs+1*8)(R29) + MOVV R10, (sysMaxArgs+2*8)(R29) + + // call the service routine + MOVV $·syscalls(SB), R26 + SLL $3, R8 + ADD R8, R26 + MOVV (R26), R26 + JAL (R26) + + // copy the return values back to the caller's stack + MOVV (sysMaxArgs+2*8)(R29), R10 + BEQ R0, R10, nothingToCopy + MOVV (sysMaxArgs+0*8)(R29), R2 // duffcopy dst + MOVV (sysMaxArgs+1*8)(R29), R1 // duffcopy src + MOVV $·duffcopy+2048(SB), R26 + SLL $1, R10 + SUB R10, R26 + JAL (R26) + +nothingToCopy: + ADD $sysMaxArgs+3*8, R29 + + JMP restore + +interrupt: + // for now only clear IE + MOVV _mcause(R29), R26 + AND $~CAUSE_IP_MASK, R26 + MOVV R26, M(C0_CAUSE) + JMP restore + +restore: + // TODO disable interrupts again + MOVV _LR(R29), R31 + MOVV _mcause(R29), R26 + MOVV R26, M(C0_CAUSE) + MOVV _mepc(R29), R26 + AND $1, R26, R27 + AND $~1, R26 // remove fromHandler flag from epc + ADD $4, R26 // don't execute exception instruction again + MOVV R26, M(C0_EPC) + ADD $excCtxSize, R29 + + BNE R27, R0, return + + MOVV $·cpu0(SB), R26 + MOVV (g_sched+gobuf_sp)(R26), R29 + MOVV (g_sched+gobuf_g)(R26), g + +return: + WORD $0x42000018 // ERET + +fatal: + BREAK + JMP -1(PC) + // func sysreset(level int, addr unsafe.Pointer) bool TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 From 2837de61c274cfcedb7052aae8556ba82924d705 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Mon, 16 Oct 2023 19:38:01 +0200 Subject: [PATCH 10/45] rework implementation of inthandler --- .../arch/r4000/systim/systim_mips64.go | 2 + src/internal/cpu/r4000/creg/creg_mips64.go | 7 + src/runtime/asm_mips64.h | 7 +- src/runtime/rt0_noos_mips64.s | 7 + src/runtime/tasker_noos_mips64.go | 33 +- src/runtime/tasker_noos_mips64.s | 360 ++++++++++++++++-- 6 files changed, 377 insertions(+), 39 deletions(-) diff --git a/src/embedded/arch/r4000/systim/systim_mips64.go b/src/embedded/arch/r4000/systim/systim_mips64.go index 7f6efc3cfb8db8..591b6fca3bed25 100644 --- a/src/embedded/arch/r4000/systim/systim_mips64.go +++ b/src/embedded/arch/r4000/systim/systim_mips64.go @@ -13,6 +13,8 @@ func Setup(clkhz int64) { hz2ns = 2 * 1e9 / clkhz creg.COMPARE.Store(0xffffffff) + creg.STATUS.SetBits(creg.IP_TIMER) + creg.CAUSE.ClearBits(creg.IP_TIMER) rtos.SetSystemTimer(nanotime, setAlarm) } diff --git a/src/internal/cpu/r4000/creg/creg_mips64.go b/src/internal/cpu/r4000/creg/creg_mips64.go index eae0004b79e4df..13a740fff27f7d 100644 --- a/src/internal/cpu/r4000/creg/creg_mips64.go +++ b/src/internal/cpu/r4000/creg/creg_mips64.go @@ -53,6 +53,13 @@ const ( CU = 0x00f << 28 // coprocessor usability ) +// Cause register bits (p.171) +const ( + IP_SW = 0x03 << 8 + IP_EXT = 0x7C << 8 + IP_TIMER = 0x80 << 8 +) + func (r CP0Reg) Load() uint32 { return mfc0(r) } func (r CP0Reg) Store(val uint32) { mtc0(r, val) } diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index d3857972573ab2..25770185afffa7 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -110,8 +110,11 @@ #define CAUSE_EXC_BREAKPOINT (9 << 2) #define CAUSE_EXC_COPROCESSOR (11 << 2) -#define CAUSE_IP_MASK (0xFF << 8) -#define CAUSE_IP7 (0x80 << 8) +// Masks for Status Registers IM bits and Cause Registers IP bits +#define INTR_MASK (0xFF << 8) +#define INTR_SW (0x03 << 8) +#define INTR_EXT (0x7C << 8) +#define INTR_TIMER (0x80 << 8) // Exception Context #define _LR (0*8) diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 73f6f91d794c86..22d1883f50edfa 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -251,6 +251,13 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 JAL runtime·newproc(SB) ADD $16, sp // pop args and LR + // enable interrupts + // TODO where to enable interupts correctly? + MOVW z0, M(C0_COMPARE) + MOVW M(C0_SR), t0 + OR $(SR_IE|INTR_SW|INTR_TIMER), t0 + MOVW t0, M(C0_SR) + // start this M JAL runtime·mstart(SB) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index 440d219a0b6ebc..a51fe95e1383b3 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -5,10 +5,23 @@ package runtime import ( + "internal/abi" + "internal/cpu/r4000/creg" "unsafe" ) -type mOS [1]uint32 +// see saveGPRS and saveFPRS +const ( + numGPRS = 27 + numFPRS = 33 +) + +type mOS struct { + gprs [numGPRS]uintptr + fprs [numFPRS]float64 + sp, fp uintptr // goroutine context + ra, epc uintptr // exception context +} var bbplayer bool // TODO move to n64 board support package @@ -42,7 +55,8 @@ func curcpuSleep() {} // //go:nosplit func (cpu *cpuctx) newwork() { - // for now only single CPU is supported + count := creg.COUNT.Load() + creg.COMPARE.Store(count + 2000) } //go:nosplit @@ -77,7 +91,10 @@ func cpuid() int { // //go:nosplit func archnewm(m *m) { - // TODO implement + m.epc = abi.FuncPCABI0(mstart) + m.sp = m.g0.stack.hi + m.fp = uintptr(unsafe.Pointer(m.g0)) + m.ra = 1 // smallCtx flag } // Run scheduler immediately or at syscall exit. It's called only just before @@ -92,10 +109,18 @@ func archnewm(m *m) { // //go:nosplit func curcpuSchedule() { + // syscall still needs to duffcopy it's result back. mark for schedule + // and call scheduler after duffcopy in ISR. curcpu().schedule = true - // TODO implement } +// only called from ISR, do not use +func enterScheduler() +func saveGPRs() +func restoreGPRs() +func saveFPRs() +func restoreFPRs() + //go:nowritebarrierrec //go:nosplit func defaultWrite(fd int, p []byte) int { diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index dccf4f6c2ec81f..96f1baffbe97a2 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -16,11 +16,33 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 // main exception handler // +// R26 and R27 are free, see runtime/asm_mips64x.s +// // interrupt exceptions are disabled by EXL=1 but other exceptions can // still occur. +// +// Only syscalls and interrupts are handled at the moment, all other exceptions +// are fatal. +// +// Called from thread: +// 1. save goroutine context in g.sched (sp, fp) +// 2. switch to ISR stack +// 3. save exception context on ISR stack (ra, epc, cause) +// 4. save thread context in m.mOS (sp, fp, ra, epc) +// 5. if interrupt goto 8, if syscall continue +// 6. do syscall +// - copy args from caller thread stack to ISR stack +// - call service routine in ISR context +// - copy result from ISR stack to caller thread stack +// 7. if the syscall called curcpuSchedule, make a call to curcpuRunScheduler +// 8. if newexe, restore context from that thread (sp, fp, ra, epc) +// 9. return execution at epc +// +// Called from handler: +// same, but skip 1, 2, 4 TEXT runtime·inthandler(SB),NOSPLIT|NOFRAME,$0 - // R26 and R27 are free, see runtime/asm_mips64x.s - // switch to special ISR goroutine +start: + // determine caller stack MOVV $·cpu0(SB), R26 BNE R26, g, fromThread MOVV $1, R27 // fromHandler flag @@ -28,25 +50,33 @@ TEXT runtime·inthandler(SB),NOSPLIT|NOFRAME,$0 fromThread: MOVV $0, R27 + + // save goroutine context in g.sched MOVV R29, (g_sched+gobuf_sp)(R26) MOVV g, (g_sched+gobuf_g)(R26) MOVV (g_stack+stack_hi)(R26), R29 + // switch to special ISR goroutine MOVV R26, g fromHandler: - // save exception context + // save exception context on ISR stack SUB $excCtxSize, R29 - MOVV R31, _LR(R29) + OR $1, R31, R26 // encode smallCtx flag in ra + MOVV R26, _LR(R29) MOVV M(C0_CAUSE), R26 MOVV R26, _mcause(R29) MOVV M(C0_EPC), R26 OR R27, R26 // encode fromHandler flag in EPC MOVV R26, _mepc(R29) - // TODO reenable interrupts? they might clobber R26, R27 + // mask interrupts + MOVV M(C0_SR), R26 + MOVV $~(INTR_SW|INTR_TIMER), R27 + AND R27, R26 + MOVV R26, M(C0_SR) // branch depending on exception cause - MOVV _mcause(R29), R26 + MOVV M(C0_CAUSE), R26 AND $CAUSE_EXC_MASK, R26 MOVV $CAUSE_EXC_SYSCALL, R27 @@ -57,27 +87,47 @@ fromHandler: JMP fatal +// System call is like oridnary function call so all registers are caller save +// (Go ABI0). Meaning we don't need to save GPRs and are free to use them. The +// tiny wrapper over SYSCALL instruction adds additional parameters in A3-A5 +// registers: +// +// R8: syscall number +// R9: argument data size on the stack (+8 for caller return address) +// R10: return data size on the stack syscall: - // R8 - syscall number, - // R9 - argument data size on the stack (+8 for frame-pointer), - // R10 - return data size on the stack. - // - // Now that we know we are in a syscall, we are free to use any - // registers. The caller should expect them to be clobbered after a - // function call. + // if fromHandler skip saving thread context + MOVV _mepc(R29), R27 + ADD $4, R27 // don't execute syscall instruction again + MOVV R27, _mepc(R29) + AND $1, R27 // fromHandler flag + BNE R27, R0, currentStack + + // save thread context in mOS + // needs to be done before the syscall, cpuctx.exe might be nil + // afterwards (e.g. in nanosleep) + // TODO skip for fast syscall + MOVV (cpuctx_exe)(g), R27 + + MOVV _LR(R29), R26 + MOVV R26, (m_mOS+mOS_ra)(R27) - // determine caller stack MOVV _mepc(R29), R26 - AND $1, R26, R27 // fromHandler flag - BNE R27, R0, currentStack + AND $~1, R26 // remove fromHandler flag from epc + MOVV R26, (m_mOS+mOS_epc)(R27) + + MOVV (g_sched+gobuf_sp)(g), R26 + MOVV R26, (m_mOS+mOS_sp)(R27) + MOVV (g_sched+gobuf_g)(g), R26 + MOVV R26, (m_mOS+mOS_fp)(R27) MOVV (g_sched+gobuf_sp)(g), R1 // duffcopy src thread - JMP continue + JMP duffcopy currentStack: ADD $excCtxSize, R29, R1 // duffcopy src handler -continue: +duffcopy: // 3 extra registers to preserve src, dst and size of result SUB $sysMaxArgs+3*8, R29 @@ -93,12 +143,24 @@ continue: MOVV R2, (sysMaxArgs+1*8)(R29) MOVV R10, (sysMaxArgs+2*8)(R29) + // reenable exceptions + MOVV M(C0_SR), R26 + AND $~SR_EXL, R26 + MOVV R26, M(C0_SR) + + // reminder: don't use R23, R26 or R27 when interrupts enabled + // call the service routine - MOVV $·syscalls(SB), R26 + MOVV $·syscalls(SB), R9 SLL $3, R8 - ADD R8, R26 - MOVV (R26), R26 - JAL (R26) + ADD R8, R9 + MOVV (R9), R9 + JAL (R9) + + // disable exceptions again + MOVV M(C0_SR), R8 + OR $SR_EXL, R8 + MOVV R8, M(C0_SR) // copy the return values back to the caller's stack MOVV (sysMaxArgs+2*8)(R29), R10 @@ -113,27 +175,105 @@ continue: nothingToCopy: ADD $sysMaxArgs+3*8, R29 - JMP restore + // run the scheduler if the syscall wants it + MOVB cpuctx_schedule(g), R8 // handlers will not set this + BEQ R8, R0, restore + JMP enterScheduler +// An interrupt was caused by timer, software or externally. This can happen on +// any instruction. We need to save all GPRs in our context. interrupt: - // for now only clear IE - MOVV _mcause(R29), R26 - AND $~CAUSE_IP_MASK, R26 - MOVV R26, M(C0_CAUSE) - JMP restore + MOVV (cpuctx_exe)(g), R26 + MOVV $(m_mOS+mOS_gprs)(R26), R26 + JAL ·saveGPRs(SB) + + // save thread context in mOS + MOVV (cpuctx_exe)(g), R27 + + MOVV _LR(R29), R26 + AND $~1, R26 // remove smallCtx flag from ra + MOVV R26, (m_mOS+mOS_ra)(R27) + + MOVV _mepc(R29), R26 + AND $~1, R26 // remove fromHandler flag from epc + MOVV R26, (m_mOS+mOS_epc)(R27) + + MOVV (g_sched+gobuf_sp)(g), R26 + MOVV R26, (m_mOS+mOS_sp)(R27) + MOVV (g_sched+gobuf_g)(g), R26 + MOVV R26, (m_mOS+mOS_fp)(R27) + + // clear all IP (interrupt pending) bits + // TODO only timer supported at the moment + MOVV M(C0_COMPARE), R26 + MOVV R26, M(C0_COMPARE) + + JMP enterScheduler + +enterScheduler: + // pop everthing from stack + ADD $excCtxSize, R29 + + // reenable interrupts + MOVV M(C0_SR), R26 + AND $~SR_EXL, R26 + MOVV R26, M(C0_SR) + + // reminder: don't use R23, R26 or R27 when interrupts enabled + + // enter scheduler + MOVB R0, cpuctx_schedule(g) + JAL ·curcpuRunScheduler(SB) + + // disable exceptions again + MOVV M(C0_SR), R8 + OR $SR_EXL, R8 + MOVV R8, M(C0_SR) + + // clear cpuctx.newexe + MOVB R0, (cpuctx_newexe)(g) + + MOVV (cpuctx_exe)(g), R27 + MOVV (m_mOS+mOS_ra)(R27), R9 + AND $~1, R9, R23 // remove smallCtx flag + AND $1, R9, R10 // smallCtx flag + BNE R0, R10, smallCtx + + MOVV $(m_mOS+mOS_gprs)(R27), R26 + JAL ·restoreGPRs(SB) +smallCtx: + // unmask interrupts + MOVV M(C0_SR), R26 + OR $(INTR_SW|INTR_TIMER), R26 + MOVV R26, M(C0_SR) + + MOVV (m_mOS+mOS_sp)(R27), R29 + MOVV (m_mOS+mOS_fp)(R27), g + MOVV R23, R31 + MOVV (m_mOS+mOS_epc)(R27), R26 + MOVV R26, M(C0_EPC) + + JMP return + +// restore ctx of caller restore: - // TODO disable interrupts again - MOVV _LR(R29), R31 - MOVV _mcause(R29), R26 - MOVV R26, M(C0_CAUSE) + MOVV _LR(R29), R26 + AND $~1, R26, R31 // remove smallCtx flag from ra +// MOVV _mcause(R29), R26 +// MOVV R26, M(C0_CAUSE) MOVV _mepc(R29), R26 AND $1, R26, R27 AND $~1, R26 // remove fromHandler flag from epc - ADD $4, R26 // don't execute exception instruction again MOVV R26, M(C0_EPC) + ADD $excCtxSize, R29 + // unmask interrupts + MOVV M(C0_SR), R26 + OR $(INTR_SW|INTR_TIMER), R26 + MOVV R26, M(C0_SR) + BNE R27, R0, return MOVV $·cpu0(SB), R26 @@ -147,6 +287,159 @@ fatal: BREAK JMP -1(PC) +// stolen from runtime/preempt_mips64x.s +// R26 must point to where gprs will be stored +TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 + MOVV R1, 0(R26) + MOVV R2, 8(R26) + MOVV R3, 16(R26) + MOVV R4, 24(R26) + MOVV R5, 32(R26) + MOVV R6, 40(R26) + MOVV R7, 48(R26) + MOVV R8, 56(R26) + MOVV R9, 64(R26) + MOVV R10, 72(R26) + MOVV R11, 80(R26) + MOVV R12, 88(R26) + MOVV R13, 96(R26) + MOVV R14, 104(R26) + MOVV R15, 112(R26) + MOVV R16, 120(R26) + MOVV R17, 128(R26) + MOVV R18, 136(R26) + MOVV R19, 144(R26) + MOVV R20, 152(R26) + MOVV R21, 160(R26) + MOVV R22, 168(R26) + MOVV R24, 176(R26) + MOVV R25, 184(R26) + MOVV RSB, 192(R26) + MOVV HI, R1 + MOVV R1, 200(R26) + MOVV LO, R1 + MOVV R1, 208(R26) + RET + +// stolen from runtime/preempt_mips64x.s +// might clobber some gprs! +// R26 must point to where fprs will be stored +TEXT ·saveFPRs(SB),NOSPLIT|NOFRAME,$0 + #ifndef GOMIPS64_softfloat + MOVV FCR31, R1 + MOVV R1, 0(R26) + MOVD F0, 8(R26) + MOVD F1, 16(R26) + MOVD F2, 24(R26) + MOVD F3, 32(R26) + MOVD F4, 40(R26) + MOVD F5, 48(R26) + MOVD F6, 56(R26) + MOVD F7, 64(R26) + MOVD F8, 72(R26) + MOVD F9, 80(R26) + MOVD F10, 88(R26) + MOVD F11, 96(R26) + MOVD F12, 104(R26) + MOVD F13, 112(R26) + MOVD F14, 120(R26) + MOVD F15, 128(R26) + MOVD F16, 136(R26) + MOVD F17, 144(R26) + MOVD F18, 152(R26) + MOVD F19, 160(R26) + MOVD F20, 168(R26) + MOVD F21, 176(R26) + MOVD F22, 184(R26) + MOVD F23, 192(R26) + MOVD F24, 200(R26) + MOVD F25, 208(R26) + MOVD F26, 216(R26) + MOVD F27, 224(R26) + MOVD F28, 232(R26) + MOVD F29, 240(R26) + MOVD F30, 248(R26) + MOVD F31, 256(R26) + #endif + RET + + +// stolen from runtime/preempt_mips64x.s +// might clobber some gprs! +// R26 must point to stored fprs +TEXT ·restoreFPRs(SB),NOSPLIT|NOFRAME,$0 + #ifndef GOMIPS64_softfloat + MOVD 256(R26), F31 + MOVD 248(R26), F30 + MOVD 240(R26), F29 + MOVD 232(R26), F28 + MOVD 224(R26), F27 + MOVD 216(R26), F26 + MOVD 208(R26), F25 + MOVD 200(R26), F24 + MOVD 192(R26), F23 + MOVD 184(R26), F22 + MOVD 176(R26), F21 + MOVD 168(R26), F20 + MOVD 160(R26), F19 + MOVD 152(R26), F18 + MOVD 144(R26), F17 + MOVD 136(R26), F16 + MOVD 128(R26), F15 + MOVD 120(R26), F14 + MOVD 112(R26), F13 + MOVD 104(R26), F12 + MOVD 96(R26), F11 + MOVD 88(R26), F10 + MOVD 80(R26), F9 + MOVD 72(R26), F8 + MOVD 64(R26), F7 + MOVD 56(R26), F6 + MOVD 48(R26), F5 + MOVD 40(R26), F4 + MOVD 32(R26), F3 + MOVD 24(R26), F2 + MOVD 16(R26), F1 + MOVD 8(R26), F0 + MOVV 0(R26), R1 + MOVV R1, FCR31 + #endif + RET + +// stolen from runtime/preempt_mips64x.s +// R26 must point to stored gprs +TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 + MOVV 208(R26), R1 + MOVV R1, LO + MOVV 200(R26), R1 + MOVV R1, HI + MOVV 192(R26), RSB + MOVV 184(R26), R25 + MOVV 176(R26), R24 + MOVV 168(R26), R22 + MOVV 160(R26), R21 + MOVV 152(R26), R20 + MOVV 144(R26), R19 + MOVV 136(R26), R18 + MOVV 128(R26), R17 + MOVV 120(R26), R16 + MOVV 112(R26), R15 + MOVV 104(R26), R14 + MOVV 96(R26), R13 + MOVV 88(R26), R12 + MOVV 80(R26), R11 + MOVV 72(R26), R10 + MOVV 64(R26), R9 + MOVV 56(R26), R8 + MOVV 48(R26), R7 + MOVV 40(R26), R6 + MOVV 32(R26), R5 + MOVV 24(R26), R4 + MOVV 16(R26), R3 + MOVV 8(R26), R2 + MOVV 0(R26), R1 + RET + // func sysreset(level int, addr unsafe.Pointer) bool TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 @@ -155,3 +448,4 @@ TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 // func syscachemaint(op int, p unsafe.Pointer, size int) TEXT ·syscachemaint(SB),NOSPLIT,$0-12 NOP // TODO + From 29c8a1960d0b25fa03dc275a3b2de10712a79dac Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Tue, 17 Oct 2023 23:20:50 +0200 Subject: [PATCH 11/45] Save a few cycles in creg access --- src/internal/cpu/r4000/creg/creg_mips64.s | 239 +++++----------------- 1 file changed, 47 insertions(+), 192 deletions(-) diff --git a/src/internal/cpu/r4000/creg/creg_mips64.s b/src/internal/cpu/r4000/creg/creg_mips64.s index e476af29d50161..c2dd1354fefd5b 100644 --- a/src/internal/cpu/r4000/creg/creg_mips64.s +++ b/src/internal/cpu/r4000/creg/creg_mips64.s @@ -3,358 +3,213 @@ TEXT ·mtc0(SB),NOSPLIT,$0-12 MOVV creg+0(FP), R4 MOVWU val+8(FP), R5 - MOVW $0, R6 - BEQ R6, R4, creg0 - MOVW $1, R6 - BEQ R6, R4, creg1 - MOVW $2, R6 - BEQ R6, R4, creg2 - MOVW $3, R6 - BEQ R6, R4, creg3 - MOVW $4, R6 - BEQ R6, R4, creg4 - MOVW $5, R6 - BEQ R6, R4, creg5 - MOVW $6, R6 - BEQ R6, R4, creg6 - MOVW $7, R6 - BEQ R6, R4, creg7 - MOVW $8, R6 - BEQ R6, R4, creg8 - MOVW $9, R6 - BEQ R6, R4, creg9 - MOVW $10, R6 - BEQ R6, R4, creg10 - MOVW $11, R6 - BEQ R6, R4, creg11 - MOVW $12, R6 - BEQ R6, R4, creg12 - MOVW $13, R6 - BEQ R6, R4, creg13 - MOVW $14, R6 - BEQ R6, R4, creg14 - MOVW $15, R6 - BEQ R6, R4, creg15 - MOVW $16, R6 - BEQ R6, R4, creg16 - MOVW $17, R6 - BEQ R6, R4, creg17 - MOVW $18, R6 - BEQ R6, R4, creg18 - MOVW $19, R6 - BEQ R6, R4, creg19 - MOVW $20, R6 - BEQ R6, R4, creg20 - MOVW $21, R6 - BEQ R6, R4, creg21 - MOVW $22, R6 - BEQ R6, R4, creg22 - MOVW $23, R6 - BEQ R6, R4, creg23 - MOVW $24, R6 - BEQ R6, R4, creg24 - MOVW $25, R6 - BEQ R6, R4, creg25 - MOVW $26, R6 - BEQ R6, R4, creg26 - MOVW $27, R6 - BEQ R6, R4, creg27 - MOVW $28, R6 - BEQ R6, R4, creg28 - MOVW $29, R6 - BEQ R6, R4, creg29 - MOVW $30, R6 - BEQ R6, R4, creg30 - MOVW $31, R6 - BEQ R6, R4, creg31 -creg0: + SLL $4, R4 + MOVV $_mtc0(SB), R6 + ADDU R6, R4 + JMP (R4) + + +TEXT _mtc0(SB),NOSPLIT|NOFRAME,$0 MOVW R5, M(0) RET -creg1: + WORD $0x0 MOVW R5, M(1) RET -creg2: + WORD $0x0 MOVW R5, M(2) RET -creg3: + WORD $0x0 MOVW R5, M(3) RET -creg4: + WORD $0x0 MOVW R5, M(4) RET -creg5: + WORD $0x0 MOVW R5, M(5) RET -creg6: + WORD $0x0 MOVW R5, M(6) RET -creg7: + WORD $0x0 MOVW R5, M(7) RET -creg8: + WORD $0x0 MOVW R5, M(8) RET -creg9: + WORD $0x0 MOVW R5, M(9) RET -creg10: + WORD $0x0 MOVW R5, M(10) RET -creg11: + WORD $0x0 MOVW R5, M(11) RET -creg12: + WORD $0x0 MOVW R5, M(12) RET -creg13: + WORD $0x0 MOVW R5, M(13) RET -creg14: + WORD $0x0 MOVW R5, M(14) RET -creg15: + WORD $0x0 MOVW R5, M(15) RET -creg16: + WORD $0x0 MOVW R5, M(16) RET -creg17: + WORD $0x0 MOVW R5, M(17) RET -creg18: + WORD $0x0 MOVW R5, M(18) RET -creg19: + WORD $0x0 MOVW R5, M(19) RET -creg20: + WORD $0x0 MOVW R5, M(20) RET -creg21: + WORD $0x0 MOVW R5, M(21) RET -creg22: + WORD $0x0 MOVW R5, M(22) RET -creg23: + WORD $0x0 MOVW R5, M(23) RET -creg24: + WORD $0x0 MOVW R5, M(24) RET -creg25: + WORD $0x0 MOVW R5, M(25) RET -creg26: + WORD $0x0 MOVW R5, M(26) RET -creg27: + WORD $0x0 MOVW R5, M(27) RET -creg28: + WORD $0x0 MOVW R5, M(28) RET -creg29: + WORD $0x0 MOVW R5, M(29) RET -creg30: + WORD $0x0 MOVW R5, M(30) RET -creg31: + WORD $0x0 MOVW R5, M(31) RET + WORD $0x0 + TEXT ·mfc0(SB),NOSPLIT,$0-12 MOVV creg+0(FP), R4 - MOVW $0, R6 - BEQ R6, R4, creg0 - MOVW $1, R6 - BEQ R6, R4, creg1 - MOVW $2, R6 - BEQ R6, R4, creg2 - MOVW $3, R6 - BEQ R6, R4, creg3 - MOVW $4, R6 - BEQ R6, R4, creg4 - MOVW $5, R6 - BEQ R6, R4, creg5 - MOVW $6, R6 - BEQ R6, R4, creg6 - MOVW $7, R6 - BEQ R6, R4, creg7 - MOVW $8, R6 - BEQ R6, R4, creg8 - MOVW $9, R6 - BEQ R6, R4, creg9 - MOVW $10, R6 - BEQ R6, R4, creg10 - MOVW $11, R6 - BEQ R6, R4, creg11 - MOVW $12, R6 - BEQ R6, R4, creg12 - MOVW $13, R6 - BEQ R6, R4, creg13 - MOVW $14, R6 - BEQ R6, R4, creg14 - MOVW $15, R6 - BEQ R6, R4, creg15 - MOVW $16, R6 - BEQ R6, R4, creg16 - MOVW $17, R6 - BEQ R6, R4, creg17 - MOVW $18, R6 - BEQ R6, R4, creg18 - MOVW $19, R6 - BEQ R6, R4, creg19 - MOVW $20, R6 - BEQ R6, R4, creg20 - MOVW $21, R6 - BEQ R6, R4, creg21 - MOVW $22, R6 - BEQ R6, R4, creg22 - MOVW $23, R6 - BEQ R6, R4, creg23 - MOVW $24, R6 - BEQ R6, R4, creg24 - MOVW $25, R6 - BEQ R6, R4, creg25 - MOVW $26, R6 - BEQ R6, R4, creg26 - MOVW $27, R6 - BEQ R6, R4, creg27 - MOVW $28, R6 - BEQ R6, R4, creg28 - MOVW $29, R6 - BEQ R6, R4, creg29 - MOVW $30, R6 - BEQ R6, R4, creg30 - MOVW $31, R6 - BEQ R6, R4, creg31 -creg0: + SLL $4, R4 + MOVV $_mfc0(SB), R6 + ADDU R6, R4 + JMP (R4) + + +TEXT _mfc0(SB),NOSPLIT|NOFRAME,$0 MOVW M(0), R5 MOVW R5, ret+8(FP) RET -creg1: MOVW M(1), R5 MOVW R5, ret+8(FP) RET -creg2: MOVW M(2), R5 MOVW R5, ret+8(FP) RET -creg3: MOVW M(3), R5 MOVW R5, ret+8(FP) RET -creg4: MOVW M(4), R5 MOVW R5, ret+8(FP) RET -creg5: MOVW M(5), R5 MOVW R5, ret+8(FP) RET -creg6: MOVW M(6), R5 MOVW R5, ret+8(FP) RET -creg7: MOVW M(7), R5 MOVW R5, ret+8(FP) RET -creg8: MOVW M(8), R5 MOVW R5, ret+8(FP) RET -creg9: MOVW M(9), R5 MOVW R5, ret+8(FP) RET -creg10: MOVW M(10), R5 MOVW R5, ret+8(FP) RET -creg11: MOVW M(11), R5 MOVW R5, ret+8(FP) RET -creg12: MOVW M(12), R5 MOVW R5, ret+8(FP) RET -creg13: MOVW M(13), R5 MOVW R5, ret+8(FP) RET -creg14: MOVW M(14), R5 MOVW R5, ret+8(FP) RET -creg15: MOVW M(15), R5 MOVW R5, ret+8(FP) RET -creg16: MOVW M(16), R5 MOVW R5, ret+8(FP) RET -creg17: MOVW M(17), R5 MOVW R5, ret+8(FP) RET -creg18: MOVW M(18), R5 MOVW R5, ret+8(FP) RET -creg19: MOVW M(19), R5 MOVW R5, ret+8(FP) RET -creg20: MOVW M(20), R5 MOVW R5, ret+8(FP) RET -creg21: MOVW M(21), R5 MOVW R5, ret+8(FP) RET -creg22: MOVW M(22), R5 MOVW R5, ret+8(FP) RET -creg23: MOVW M(23), R5 MOVW R5, ret+8(FP) RET -creg24: MOVW M(24), R5 MOVW R5, ret+8(FP) RET -creg25: MOVW M(25), R5 MOVW R5, ret+8(FP) RET -creg26: MOVW M(26), R5 MOVW R5, ret+8(FP) RET -creg27: MOVW M(27), R5 MOVW R5, ret+8(FP) RET -creg28: MOVW M(28), R5 MOVW R5, ret+8(FP) RET -creg29: MOVW M(29), R5 MOVW R5, ret+8(FP) RET -creg30: MOVW M(30), R5 MOVW R5, ret+8(FP) RET -creg31: MOVW M(31), R5 MOVW R5, ret+8(FP) RET From 7b524bdefd16a5af6dadb80a2e84ea35d0e3c5e6 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Fri, 20 Oct 2023 00:04:31 +0200 Subject: [PATCH 12/45] Use SW interrupts for signaling newwork() The COMPARE register was only used because of a bug in MAME which doesn't exist anymore in newer versions. Moreover this implementation could result in a deadlock because the scheduler also sets this register. --- src/runtime/tasker_noos_mips64.go | 4 ++-- src/runtime/tasker_noos_mips64.s | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index a51fe95e1383b3..c56fe432809683 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -55,8 +55,8 @@ func curcpuSleep() {} // //go:nosplit func (cpu *cpuctx) newwork() { - count := creg.COUNT.Load() - creg.COMPARE.Store(count + 2000) + // TODO use only one of the two interrupts + creg.CAUSE.SetBits(creg.IP_SW) } //go:nosplit diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 96f1baffbe97a2..7bfcf554206189 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -204,7 +204,9 @@ interrupt: MOVV R26, (m_mOS+mOS_fp)(R27) // clear all IP (interrupt pending) bits - // TODO only timer supported at the moment + MOVV M(C0_CAUSE), R26 + AND $~INTR_MASK, R26 + MOVV R26, M(C0_CAUSE) MOVV M(C0_COMPARE), R26 MOVV R26, M(C0_COMPARE) From eb3e45053dce153703e999ca7ae6ff5d0303b13c Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Fri, 20 Oct 2023 00:07:34 +0200 Subject: [PATCH 13/45] Simple default write for debugging --- src/runtime/tasker_noos_mips64.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index c56fe432809683..bd325d28dac209 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -121,9 +121,22 @@ func restoreGPRs() func saveFPRs() func restoreFPRs() +var lastlog [1024 * 16]byte +var lastidx = 0 + +// Just write into some ringbuffer for now, which can be inspected in a memory +// dump. +// //go:nowritebarrierrec //go:nosplit func defaultWrite(fd int, p []byte) int { + for i := 0; i < len(p); i++ { + lastlog[lastidx] = p[i] + lastidx += 1 + if lastidx >= len(lastlog) { + lastidx = 0 + } + } return len(p) } From b1021cafeaca2d6e07041c1433363973dff9d6d8 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sun, 22 Oct 2023 07:42:11 +0200 Subject: [PATCH 14/45] Support user interrupt vectors --- src/cmd/link/internal/ld/deadcode.go | 5 +++ src/cmd/link/internal/mips64/asm.go | 39 ++++++++++++++++++- src/runtime/tasker_noos_mips64.s | 58 ++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go index aff7a24ea50e01..f82dd7126e61f3 100644 --- a/src/cmd/link/internal/ld/deadcode.go +++ b/src/cmd/link/internal/ld/deadcode.go @@ -90,6 +90,11 @@ func (d *deadcodePass) init() { first, last = -14, 479 case sys.RISCV64: first, last = 1, 1023 + case sys.MIPS64: + first, last = 1, 5 + // runtime.unhandledExternalInterrupt is also a handler. + // Must be called somewhere in the code, otherwise it + // gets removed without error and relocs point to 0x0. } for i := first; i <= last; i++ { names = append(names, InterruptHandler(i)) diff --git a/src/cmd/link/internal/mips64/asm.go b/src/cmd/link/internal/mips64/asm.go index 38c486b6bb70d3..d690a626dba6a0 100644 --- a/src/cmd/link/internal/mips64/asm.go +++ b/src/cmd/link/internal/mips64/asm.go @@ -72,8 +72,45 @@ func lookupFuncSym(ldr *loader.Loader, name string) loader.Sym { } func gentext_noos(ctxt *ld.Link, ldr *loader.Loader) { - // TODO support interrupt vector table + vectors := ldr.CreateSymForUpdate("runtime.vectors", sym.SymVerABI0) + vectors.SetType(sym.STEXT) + vectors.SetReachable(true) + vectors.SetAlign(8) + + unhandledInterrupt := ldr.Lookup("runtime.unhandledExternalInterrupt", sym.SymVerABI0) + if unhandledInterrupt == 0 { + ld.Errorf(nil, "runtime.unhandledExternalInterrupt not defined") + } + + // search for user defined ISRs: //go:linkname functionName IRQ%d_Handler + // This code tries to keep runtime.vectors small, by cutting off all irq + // at the end that point to runtime.unhandledExternalInterrupt. + var irqHandlers [5]loader.Sym + irqNum := 1 + for i := 1; i < len(irqHandlers); i++ { + s := lookupFuncSym(ldr, ld.InterruptHandler(i)) + if s == 0 { + irqHandlers[i] = unhandledInterrupt + } else { + irqHandlers[i] = s + irqNum = i + 1 + } + } + + vectors.AddUint64(ctxt.Arch, uint64(irqNum)) + relocs := vectors.AddRelocs(irqNum - 1) + for i, s := range irqHandlers[1:irqNum] { + ldr.MakeSymbolUpdater(s).SetReachable(true) + rel := relocs.At(i) + rel.SetSym(s) + rel.SetType(objabi.R_ADDR) + rel.SetSiz(8) + rel.SetOff(int32(vectors.AddUint64(ctxt.Arch, 0))) + } + + ctxt.Textp = append(ctxt.Textp, vectors.Sym()) + // move the entry symbol at the beggining of the text segment entry := lookupFuncSym(ldr, *ld.FlagEntrySymbol) if entry == 0 { ld.Errorf(nil, "cannot find entry function: %s", *ld.FlagEntrySymbol) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 7bfcf554206189..b9435efcfb48d7 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -71,7 +71,7 @@ fromHandler: // mask interrupts MOVV M(C0_SR), R26 - MOVV $~(INTR_SW|INTR_TIMER), R27 + MOVV $~(INTR_SW|INTR_EXT|INTR_TIMER), R27 AND R27, R26 MOVV R26, M(C0_SR) @@ -205,6 +205,11 @@ interrupt: // clear all IP (interrupt pending) bits MOVV M(C0_CAUSE), R26 + AND $INTR_EXT, R26, R27 + BNE R27, R0, externalInterrupt + + // must be a timer or software interrupt. In both cases clear pending + // bits and enter the scheduler AND $~INTR_MASK, R26 MOVV R26, M(C0_CAUSE) MOVV M(C0_COMPARE), R26 @@ -212,11 +217,51 @@ interrupt: JMP enterScheduler +externalInterrupt: + // external interrupts are handled by the application. We need to call + // one of the registered handlers + // TODO save gprs on stack? see tasker_noos_riscv64.s + SRL $8, R27 // IP_EXT + MOVV $1, R8 + +loop: + // find and handle only the first pending interrupt. If there are + // multiple interrupts pending, the interrupt handler will run again + AND $1, R27, R9 + BNE R9, R0, callVector + ADD $1, R8 + SRL $1, R27 + JMP loop + +callVector: + SLL $3, R8, R9 // irq vector offset + + // TODO allow nested interrupts? see tasker_noos_riscv64.s + + // get interrupt vector + MOVV $runtime·vectors(SB), R26 + MOVV (R26), R10 + SUB R8, R10 + BLEZ R10, noHandler + ADD R9, R26 + MOVV (R26), R26 + + JAL (R26) + + // external interrupts can't be cleared by clearing the bits in cause + // register. The user handler must have cleared the interrupt at this + // point in the external source, probably via mmio. + + JMP enterScheduler //TODO no? + +noHandler: + JMP ·unhandledExternalInterrupt(SB) + enterScheduler: // pop everthing from stack ADD $excCtxSize, R29 - // reenable interrupts + // reenable exceptions MOVV M(C0_SR), R26 AND $~SR_EXL, R26 MOVV R26, M(C0_SR) @@ -247,7 +292,7 @@ enterScheduler: smallCtx: // unmask interrupts MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_TIMER), R26 + OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 MOVV R26, M(C0_SR) MOVV (m_mOS+mOS_sp)(R27), R29 @@ -273,7 +318,7 @@ restore: // unmask interrupts MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_TIMER), R26 + OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 MOVV R26, M(C0_SR) BNE R27, R0, return @@ -289,6 +334,11 @@ fatal: BREAK JMP -1(PC) +// Required by the linker, runtime.vectors will be initialized with this. +TEXT runtime·unhandledExternalInterrupt(SB),NOSPLIT|NOFRAME,$0 + BREAK + JMP -1(PC) + // stolen from runtime/preempt_mips64x.s // R26 must point to where gprs will be stored TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 From 06704ea877ac11337b8876206dc201870f796c31 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Tue, 24 Oct 2023 09:07:34 +0200 Subject: [PATCH 15/45] Fix hardware interrupt handler Don't call the scheduler and enable exceptions in the user handler, it might call the scheduler. --- src/runtime/tasker_noos_mips64.s | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index b9435efcfb48d7..8beda0edb49a68 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -40,6 +40,12 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 // // Called from handler: // same, but skip 1, 2, 4 +// +// Other basic things I learned while writing this: +// - If we want to support nested exceptions, save context must be on the stack. +// If we want to schedule, context must be in the m object. +// - There are no callee save registers in go. Assume all gprs clobbered after +// a function call. TEXT runtime·inthandler(SB),NOSPLIT|NOFRAME,$0 start: // determine caller stack @@ -89,7 +95,7 @@ fromHandler: // System call is like oridnary function call so all registers are caller save // (Go ABI0). Meaning we don't need to save GPRs and are free to use them. The -// tiny wrapper over SYSCALL instruction adds additional parameters in A3-A5 +// tiny wrapper over SYSCALL instruction adds additional parameters in R8-R10 // registers: // // R8: syscall number @@ -203,7 +209,6 @@ interrupt: MOVV (g_sched+gobuf_g)(g), R26 MOVV R26, (m_mOS+mOS_fp)(R27) - // clear all IP (interrupt pending) bits MOVV M(C0_CAUSE), R26 AND $INTR_EXT, R26, R27 BNE R27, R0, externalInterrupt @@ -220,7 +225,6 @@ interrupt: externalInterrupt: // external interrupts are handled by the application. We need to call // one of the registered handlers - // TODO save gprs on stack? see tasker_noos_riscv64.s SRL $8, R27 // IP_EXT MOVV $1, R8 @@ -236,7 +240,9 @@ loop: callVector: SLL $3, R8, R9 // irq vector offset - // TODO allow nested interrupts? see tasker_noos_riscv64.s + // TODO allow nested interrupts. seems to be necessary for e.g. + // note.wakeup to work properly from an inthandler. Make sure to put + // context on stack instead of mOS for nested interrupts. // get interrupt vector MOVV $runtime·vectors(SB), R26 @@ -246,13 +252,26 @@ callVector: ADD R9, R26 MOVV (R26), R26 + // reenable exceptions + MOVV M(C0_SR), R27 + AND $~SR_EXL, R27 + MOVV R27, M(C0_SR) + JAL (R26) + // disable exceptions again + MOVV M(C0_SR), R8 + OR $SR_EXL, R8 + MOVV R8, M(C0_SR) + // external interrupts can't be cleared by clearing the bits in cause // register. The user handler must have cleared the interrupt at this // point in the external source, probably via mmio. - JMP enterScheduler //TODO no? + // pop everthing from stack + ADD $excCtxSize, R29 + + JMP skipScheduler noHandler: JMP ·unhandledExternalInterrupt(SB) @@ -280,6 +299,7 @@ enterScheduler: // clear cpuctx.newexe MOVB R0, (cpuctx_newexe)(g) +skipScheduler: MOVV (cpuctx_exe)(g), R27 MOVV (m_mOS+mOS_ra)(R27), R9 AND $~1, R9, R23 // remove smallCtx flag From 6516ccad5c38e994aed5b18c68404ad267b28d81 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Tue, 24 Oct 2023 21:09:57 +0200 Subject: [PATCH 16/45] Refactor inthandler in preparation for nested IRQ This commit breaks external interrupt handling. Will be fixed in the next one. --- src/runtime/tasker_noos_mips64.s | 274 ++++++++++++++++--------------- 1 file changed, 145 insertions(+), 129 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 8beda0edb49a68..2124667257adc7 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -10,9 +10,10 @@ #include "asm_mips64.h" #define sysMaxArgs (48+8) +#define ERET WORD $0x42000018 TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 - JMP ·inthandler(SB) + JMP ·exceptionTrap(SB) // main exception handler // @@ -41,13 +42,18 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 // Called from handler: // same, but skip 1, 2, 4 // +// --external-------ERET +// / +// exception---interrupt--SW----scheduler--ERET +// \ / +// --syscall------ +// // Other basic things I learned while writing this: // - If we want to support nested exceptions, save context must be on the stack. // If we want to schedule, context must be in the m object. // - There are no callee save registers in go. Assume all gprs clobbered after // a function call. -TEXT runtime·inthandler(SB),NOSPLIT|NOFRAME,$0 -start: +TEXT runtime·exceptionTrap(SB),NOSPLIT|NOFRAME,$0 // determine caller stack MOVV $·cpu0(SB), R26 BNE R26, g, fromThread @@ -61,6 +67,7 @@ fromThread: MOVV R29, (g_sched+gobuf_sp)(R26) MOVV g, (g_sched+gobuf_g)(R26) MOVV (g_stack+stack_hi)(R26), R29 + // switch to special ISR goroutine MOVV R26, g @@ -86,12 +93,17 @@ fromHandler: AND $CAUSE_EXC_MASK, R26 MOVV $CAUSE_EXC_SYSCALL, R27 - BEQ R26, R27, syscall + BNE R26, R27, 2(PC) + JMP ·syscallHandler(SB) MOVV $CAUSE_EXC_INTERRUPT, R27 - BEQ R26, R27, interrupt + BNE R26, R27, 2(PC) + JMP ·interruptHandler(SB) + + // unhandled exception + BREAK + JMP -1(PC) - JMP fatal // System call is like oridnary function call so all registers are caller save // (Go ABI0). Meaning we don't need to save GPRs and are free to use them. The @@ -101,7 +113,7 @@ fromHandler: // R8: syscall number // R9: argument data size on the stack (+8 for caller return address) // R10: return data size on the stack -syscall: +TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 // if fromHandler skip saving thread context MOVV _mepc(R29), R27 ADD $4, R27 // don't execute syscall instruction again @@ -183,12 +195,39 @@ nothingToCopy: // run the scheduler if the syscall wants it MOVB cpuctx_schedule(g), R8 // handlers will not set this - BEQ R8, R0, restore - JMP enterScheduler + BEQ R8, R0, 2(PC) + JMP ·enterScheduler(SB) + +// restore ctx of caller + MOVV _LR(R29), R26 + AND $~1, R26, R31 // remove smallCtx flag from ra +// MOVV _mcause(R29), R26 +// MOVV R26, M(C0_CAUSE) + MOVV _mepc(R29), R26 + AND $1, R26, R27 + AND $~1, R26 // remove fromHandler flag from epc + MOVV R26, M(C0_EPC) + + ADD $excCtxSize, R29 + + // unmask interrupts + MOVV M(C0_SR), R26 + OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 + MOVV R26, M(C0_SR) + + BNE R27, R0, return + + MOVV $·cpu0(SB), R26 + MOVV (g_sched+gobuf_sp)(R26), R29 + MOVV (g_sched+gobuf_g)(R26), g + +return: + ERET + // An interrupt was caused by timer, software or externally. This can happen on // any instruction. We need to save all GPRs in our context. -interrupt: +TEXT runtime·interruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (cpuctx_exe)(g), R26 MOVV $(m_mOS+mOS_gprs)(R26), R26 JAL ·saveGPRs(SB) @@ -211,7 +250,8 @@ interrupt: MOVV M(C0_CAUSE), R26 AND $INTR_EXT, R26, R27 - BNE R27, R0, externalInterrupt + BEQ R27, R0, 2(PC) + JMP ·externalInterrupt(SB) // must be a timer or software interrupt. In both cases clear pending // bits and enter the scheduler @@ -220,22 +260,72 @@ interrupt: MOVV M(C0_COMPARE), R26 MOVV R26, M(C0_COMPARE) - JMP enterScheduler + JMP ·enterScheduler(SB) + + +TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 + // pop everthing from stack + ADD $excCtxSize, R29 + + // reenable exceptions + MOVV M(C0_SR), R26 + AND $~SR_EXL, R26 + MOVV R26, M(C0_SR) + + // reminder: don't use R23, R26 or R27 when interrupts enabled + + // enter scheduler + MOVB R0, cpuctx_schedule(g) + JAL ·curcpuRunScheduler(SB) + + // disable exceptions again + MOVV M(C0_SR), R8 + OR $SR_EXL, R8 + MOVV R8, M(C0_SR) + + // clear cpuctx.newexe + MOVB R0, (cpuctx_newexe)(g) + + MOVV (cpuctx_exe)(g), R27 + MOVV (m_mOS+mOS_ra)(R27), R9 + AND $~1, R9, R23 // remove smallCtx flag + AND $1, R9, R10 // smallCtx flag + BNE R0, R10, smallCtx + + MOVV $(m_mOS+mOS_gprs)(R27), R26 + JAL ·restoreGPRs(SB) + +smallCtx: + // unmask interrupts + MOVV M(C0_SR), R26 + OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 + MOVV R26, M(C0_SR) + + MOVV (m_mOS+mOS_sp)(R27), R29 + MOVV (m_mOS+mOS_fp)(R27), g + MOVV R23, R31 + MOVV (m_mOS+mOS_epc)(R27), R26 + MOVV R26, M(C0_EPC) + + ERET + + +TEXT runtime·externalInterrupt(SB),NOSPLIT|NOFRAME,$0 + // TODO save ctx -externalInterrupt: // external interrupts are handled by the application. We need to call // one of the registered handlers SRL $8, R27 // IP_EXT MOVV $1, R8 -loop: +findVector: // find and handle only the first pending interrupt. If there are // multiple interrupts pending, the interrupt handler will run again AND $1, R27, R9 BNE R9, R0, callVector ADD $1, R8 SRL $1, R27 - JMP loop + JMP findVector callVector: SLL $3, R8, R9 // irq vector offset @@ -271,94 +361,18 @@ callVector: // pop everthing from stack ADD $excCtxSize, R29 - JMP skipScheduler + // TODO restore ctx noHandler: JMP ·unhandledExternalInterrupt(SB) -enterScheduler: - // pop everthing from stack - ADD $excCtxSize, R29 - - // reenable exceptions - MOVV M(C0_SR), R26 - AND $~SR_EXL, R26 - MOVV R26, M(C0_SR) - - // reminder: don't use R23, R26 or R27 when interrupts enabled - - // enter scheduler - MOVB R0, cpuctx_schedule(g) - JAL ·curcpuRunScheduler(SB) - - // disable exceptions again - MOVV M(C0_SR), R8 - OR $SR_EXL, R8 - MOVV R8, M(C0_SR) - - // clear cpuctx.newexe - MOVB R0, (cpuctx_newexe)(g) - -skipScheduler: - MOVV (cpuctx_exe)(g), R27 - MOVV (m_mOS+mOS_ra)(R27), R9 - AND $~1, R9, R23 // remove smallCtx flag - AND $1, R9, R10 // smallCtx flag - BNE R0, R10, smallCtx - - MOVV $(m_mOS+mOS_gprs)(R27), R26 - JAL ·restoreGPRs(SB) - -smallCtx: - // unmask interrupts - MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 - MOVV R26, M(C0_SR) - - MOVV (m_mOS+mOS_sp)(R27), R29 - MOVV (m_mOS+mOS_fp)(R27), g - MOVV R23, R31 - MOVV (m_mOS+mOS_epc)(R27), R26 - MOVV R26, M(C0_EPC) - - JMP return - -// restore ctx of caller -restore: - MOVV _LR(R29), R26 - AND $~1, R26, R31 // remove smallCtx flag from ra -// MOVV _mcause(R29), R26 -// MOVV R26, M(C0_CAUSE) - MOVV _mepc(R29), R26 - AND $1, R26, R27 - AND $~1, R26 // remove fromHandler flag from epc - MOVV R26, M(C0_EPC) - - ADD $excCtxSize, R29 - // unmask interrupts - MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 - MOVV R26, M(C0_SR) - - BNE R27, R0, return - - MOVV $·cpu0(SB), R26 - MOVV (g_sched+gobuf_sp)(R26), R29 - MOVV (g_sched+gobuf_g)(R26), g - -return: - WORD $0x42000018 // ERET - -fatal: - BREAK - JMP -1(PC) - -// Required by the linker, runtime.vectors will be initialized with this. +// Required by the linker, runtime.vectors will default to this. TEXT runtime·unhandledExternalInterrupt(SB),NOSPLIT|NOFRAME,$0 BREAK JMP -1(PC) + // stolen from runtime/preempt_mips64x.s // R26 must point to where gprs will be stored TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 @@ -393,6 +407,42 @@ TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 MOVV R1, 208(R26) RET + +// stolen from runtime/preempt_mips64x.s +// R26 must point to stored gprs +TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 + MOVV 208(R26), R1 + MOVV R1, LO + MOVV 200(R26), R1 + MOVV R1, HI + MOVV 192(R26), RSB + MOVV 184(R26), R25 + MOVV 176(R26), R24 + MOVV 168(R26), R22 + MOVV 160(R26), R21 + MOVV 152(R26), R20 + MOVV 144(R26), R19 + MOVV 136(R26), R18 + MOVV 128(R26), R17 + MOVV 120(R26), R16 + MOVV 112(R26), R15 + MOVV 104(R26), R14 + MOVV 96(R26), R13 + MOVV 88(R26), R12 + MOVV 80(R26), R11 + MOVV 72(R26), R10 + MOVV 64(R26), R9 + MOVV 56(R26), R8 + MOVV 48(R26), R7 + MOVV 40(R26), R6 + MOVV 32(R26), R5 + MOVV 24(R26), R4 + MOVV 16(R26), R3 + MOVV 8(R26), R2 + MOVV 0(R26), R1 + RET + + // stolen from runtime/preempt_mips64x.s // might clobber some gprs! // R26 must point to where fprs will be stored @@ -478,46 +528,12 @@ TEXT ·restoreFPRs(SB),NOSPLIT|NOFRAME,$0 #endif RET -// stolen from runtime/preempt_mips64x.s -// R26 must point to stored gprs -TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 - MOVV 208(R26), R1 - MOVV R1, LO - MOVV 200(R26), R1 - MOVV R1, HI - MOVV 192(R26), RSB - MOVV 184(R26), R25 - MOVV 176(R26), R24 - MOVV 168(R26), R22 - MOVV 160(R26), R21 - MOVV 152(R26), R20 - MOVV 144(R26), R19 - MOVV 136(R26), R18 - MOVV 128(R26), R17 - MOVV 120(R26), R16 - MOVV 112(R26), R15 - MOVV 104(R26), R14 - MOVV 96(R26), R13 - MOVV 88(R26), R12 - MOVV 80(R26), R11 - MOVV 72(R26), R10 - MOVV 64(R26), R9 - MOVV 56(R26), R8 - MOVV 48(R26), R7 - MOVV 40(R26), R6 - MOVV 32(R26), R5 - MOVV 24(R26), R4 - MOVV 16(R26), R3 - MOVV 8(R26), R2 - MOVV 0(R26), R1 - RET - // func sysreset(level int, addr unsafe.Pointer) bool TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 NOP // TODO + // func syscachemaint(op int, p unsafe.Pointer, size int) TEXT ·syscachemaint(SB),NOSPLIT,$0-12 NOP // TODO - From 4dbf8e650a99321f3f50b2ff0bde832f191c853c Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 25 Oct 2023 00:06:17 +0200 Subject: [PATCH 17/45] Reimplement context saving for external irqs --- src/runtime/asm_mips64.h | 6 -- src/runtime/tasker_noos_mips64.s | 150 +++++++++++++++++++++---------- 2 files changed, 102 insertions(+), 54 deletions(-) diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index 25770185afffa7..110e7995467f86 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -116,12 +116,6 @@ #define INTR_EXT (0x7C << 8) #define INTR_TIMER (0x80 << 8) -// Exception Context -#define _LR (0*8) -#define _mcause (1*8) -#define _mepc (2*8) -#define excCtxSize (3*8) - /* Standard (R4000) cache operations. Taken from "MIPS R4000 Microprocessor User's Manual" 2nd edition: */ diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 2124667257adc7..b55fe9d5802ff7 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -12,6 +12,13 @@ #define sysMaxArgs (48+8) #define ERET WORD $0x42000018 +// Exception Context +#define _LR (0*8) +#define _mcause (1*8) +#define _mepc (2*8) +#define excCtxSize (3*8) + + TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 JMP ·exceptionTrap(SB) @@ -84,7 +91,7 @@ fromHandler: // mask interrupts MOVV M(C0_SR), R26 - MOVV $~(INTR_SW|INTR_EXT|INTR_TIMER), R27 + MOVV $~(INTR_SW|INTR_EXT), R27 AND R27, R26 MOVV R26, M(C0_SR) @@ -97,9 +104,20 @@ fromHandler: JMP ·syscallHandler(SB) MOVV $CAUSE_EXC_INTERRUPT, R27 - BNE R26, R27, 2(PC) - JMP ·interruptHandler(SB) + BNE R26, R27, fatal + + MOVV M(C0_CAUSE), R26 + AND $INTR_MASK, R26 + + AND $INTR_EXT, R26, R27 + BEQ R27, R0, 2(PC) + JMP ·externalInterruptHandler(SB) + + AND $INTR_SW, R26, R27 + BEQ R27, R0, 2(PC) + JMP ·softwareInterruptHandler(SB) +fatal: // unhandled exception BREAK JMP -1(PC) @@ -212,7 +230,7 @@ nothingToCopy: // unmask interrupts MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 + OR $(INTR_SW|INTR_EXT), R26 MOVV R26, M(C0_SR) BNE R27, R0, return @@ -225,9 +243,10 @@ return: ERET -// An interrupt was caused by timer, software or externally. This can happen on +// An interrupt was caused by software. This can happen on // any instruction. We need to save all GPRs in our context. -TEXT runtime·interruptHandler(SB),NOSPLIT|NOFRAME,$0 +// TODO do we really need to save full context? SW_INT is only set in newwork() +TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (cpuctx_exe)(g), R26 MOVV $(m_mOS+mOS_gprs)(R26), R26 JAL ·saveGPRs(SB) @@ -248,17 +267,13 @@ TEXT runtime·interruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (g_sched+gobuf_g)(g), R26 MOVV R26, (m_mOS+mOS_fp)(R27) - MOVV M(C0_CAUSE), R26 - AND $INTR_EXT, R26, R27 - BEQ R27, R0, 2(PC) - JMP ·externalInterrupt(SB) - // must be a timer or software interrupt. In both cases clear pending // bits and enter the scheduler + MOVV M(C0_CAUSE), R26 AND $~INTR_MASK, R26 MOVV R26, M(C0_CAUSE) - MOVV M(C0_COMPARE), R26 - MOVV R26, M(C0_COMPARE) +// MOVV M(C0_COMPARE), R26 +// MOVV R26, M(C0_COMPARE) JMP ·enterScheduler(SB) @@ -290,15 +305,15 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 MOVV (m_mOS+mOS_ra)(R27), R9 AND $~1, R9, R23 // remove smallCtx flag AND $1, R9, R10 // smallCtx flag - BNE R0, R10, smallCtx + BNE R0, R10, smallCtx // TODO should never happen MOVV $(m_mOS+mOS_gprs)(R27), R26 - JAL ·restoreGPRs(SB) + JAL ·restoreGPRs(SB) // TODO skip if we are coming from syscall smallCtx: // unmask interrupts MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_EXT|INTR_TIMER), R26 + OR $(INTR_SW|INTR_EXT), R26 MOVV R26, M(C0_SR) MOVV (m_mOS+mOS_sp)(R27), R29 @@ -310,60 +325,99 @@ smallCtx: ERET -TEXT runtime·externalInterrupt(SB),NOSPLIT|NOFRAME,$0 - // TODO save ctx +TEXT runtime·externalInterruptHandler(SB),NOSPLIT|NOFRAME,$0 + // External interrupt can happen anytime, save full context. Context + // will be saved on the stack, to allow nested interrupts be supported. + // TODO nested interrupts aren't enabled as of now. Do we need them? + // Keep in mind that the scheduler must not be called if context is + // saved on the stack. + SUB $const_numGPRS*8, R29 + MOVV R29, R26 + JAL ·saveGPRs(SB) + + // Context is saved. Reenable exceptions. Don't use R23, R26, R27. + MOVV M(C0_SR), R27 + AND $~SR_EXL, R27 + MOVV R27, M(C0_SR) - // external interrupts are handled by the application. We need to call + // External interrupts are handled by the application. We need to call // one of the registered handlers - SRL $8, R27 // IP_EXT - MOVV $1, R8 + MOVV M(C0_CAUSE), R8 + AND $INTR_EXT, R8 + + SRL $8, R8 // INTR_EXT in lsb + MOVV $1, R9 findVector: // find and handle only the first pending interrupt. If there are // multiple interrupts pending, the interrupt handler will run again - AND $1, R27, R9 - BNE R9, R0, callVector - ADD $1, R8 - SRL $1, R27 + AND $1, R8, R10 + BNE R10, R0, callVector + ADD $1, R9 + SRL $1, R8 JMP findVector callVector: - SLL $3, R8, R9 // irq vector offset - - // TODO allow nested interrupts. seems to be necessary for e.g. - // note.wakeup to work properly from an inthandler. Make sure to put - // context on stack instead of mOS for nested interrupts. + SLL $3, R9, R8 // irq vector offset // get interrupt vector - MOVV $runtime·vectors(SB), R26 - MOVV (R26), R10 - SUB R8, R10 - BLEZ R10, noHandler - ADD R9, R26 - MOVV (R26), R26 + MOVV $runtime·vectors(SB), R10 + MOVV (R10), R11 + SUB R9, R11 + BLEZ R11, fatal + ADD R8, R10 + MOVV (R10), R10 - // reenable exceptions - MOVV M(C0_SR), R27 - AND $~SR_EXL, R27 - MOVV R27, M(C0_SR) + // Allow nested interrupts +// MOVV M(C0_SR), R8 +// OR $INTR_SW, R8 +// MOVV R8, M(C0_SR) - JAL (R26) + JAL (R10) - // disable exceptions again +// MOVV M(C0_SR), R8 +// AND $~INTR_SW, R8 +// MOVV R8, M(C0_SR) + + // External interrupts can't be cleared by clearing the bits in cause + // register. The user handler must have cleared the interrupt at this + // point in the external source, probably via mmio. + + // Disable exceptions again. Restore Context. MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) - // external interrupts can't be cleared by clearing the bits in cause - // register. The user handler must have cleared the interrupt at this - // point in the external source, probably via mmio. + // Unmask interrupts + MOVV M(C0_SR), R26 + OR $(INTR_SW|INTR_EXT), R26 + MOVV R26, M(C0_SR) + + // Restore ctx of caller + MOVV R29, R26 + JAL ·restoreGPRs(SB) + ADD $const_numGPRS*8, R29 + + MOVV _LR(R29), R26 + AND $~1, R26, R31 // Remove smallCtx flag from RA + MOVV _mepc(R29), R26 + AND $1, R26, R27 + AND $~1, R26 // Remove fromHandler flag from EPC + MOVV R26, M(C0_EPC) - // pop everthing from stack ADD $excCtxSize, R29 - // TODO restore ctx + // Don't switch stacks yet if we were called from handler + BNE R27, R0, return + + MOVV $·cpu0(SB), R26 + MOVV (g_sched+gobuf_sp)(R26), R29 + MOVV (g_sched+gobuf_g)(R26), g + +return: + ERET -noHandler: +fatal: JMP ·unhandledExternalInterrupt(SB) From 287f9a88aeb6fd578f4039f0452950aa5aa02360 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 25 Oct 2023 00:39:26 +0200 Subject: [PATCH 18/45] Fix save/restoreGPRs by including R23 --- src/runtime/tasker_noos_mips64.go | 2 +- src/runtime/tasker_noos_mips64.s | 48 +++++++++++++++++++------------ 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index bd325d28dac209..d573a5a536e457 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -12,7 +12,7 @@ import ( // see saveGPRS and saveFPRS const ( - numGPRS = 27 + numGPRS = 28 numFPRS = 33 ) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index b55fe9d5802ff7..4a04ca730a710d 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -184,7 +184,7 @@ duffcopy: AND $~SR_EXL, R26 MOVV R26, M(C0_SR) - // reminder: don't use R23, R26 or R27 when interrupts enabled + // reminder: don't use R26 or R27 when interrupts enabled // call the service routine MOVV $·syscalls(SB), R9 @@ -287,7 +287,7 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 AND $~SR_EXL, R26 MOVV R26, M(C0_SR) - // reminder: don't use R23, R26 or R27 when interrupts enabled + // reminder: don't use R26 or R27 when interrupts enabled // enter scheduler MOVB R0, cpuctx_schedule(g) @@ -304,6 +304,8 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 MOVV (cpuctx_exe)(g), R27 MOVV (m_mOS+mOS_ra)(R27), R9 AND $~1, R9, R23 // remove smallCtx flag + SUB $8, R29 + MOVV R23, 0(R29) AND $1, R9, R10 // smallCtx flag BNE R0, R10, smallCtx // TODO should never happen @@ -316,9 +318,11 @@ smallCtx: OR $(INTR_SW|INTR_EXT), R26 MOVV R26, M(C0_SR) + MOVV 0(R29), R26 + ADD $8, R29 MOVV (m_mOS+mOS_sp)(R27), R29 MOVV (m_mOS+mOS_fp)(R27), g - MOVV R23, R31 + MOVV R26, R31 MOVV (m_mOS+mOS_epc)(R27), R26 MOVV R26, M(C0_EPC) @@ -335,7 +339,7 @@ TEXT runtime·externalInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV R29, R26 JAL ·saveGPRs(SB) - // Context is saved. Reenable exceptions. Don't use R23, R26, R27. + // Context is saved. Reenable exceptions. Don't use R26, R27. MOVV M(C0_SR), R27 AND $~SR_EXL, R27 MOVV R27, M(C0_SR) @@ -399,11 +403,15 @@ callVector: ADD $const_numGPRS*8, R29 MOVV _LR(R29), R26 - AND $~1, R26, R31 // Remove smallCtx flag from RA + MOVV $~1, R27 + AND R27, R26, R31 // Remove smallCtx flag from RA MOVV _mepc(R29), R26 - AND $1, R26, R27 - AND $~1, R26 // Remove fromHandler flag from EPC - MOVV R26, M(C0_EPC) + MOVV $~1, R27 + AND R26, R27 // Remove fromHandler flag from EPC + MOVV R27, M(C0_EPC) + + MOVV $1, R27 + AND R26, R27 ADD $excCtxSize, R29 @@ -428,6 +436,7 @@ TEXT runtime·unhandledExternalInterrupt(SB),NOSPLIT|NOFRAME,$0 // stolen from runtime/preempt_mips64x.s +// TODO why was temp register (R23) not included in preempt_mips64x? // R26 must point to where gprs will be stored TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 MOVV R1, 0(R26) @@ -452,26 +461,29 @@ TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 MOVV R20, 152(R26) MOVV R21, 160(R26) MOVV R22, 168(R26) - MOVV R24, 176(R26) - MOVV R25, 184(R26) - MOVV RSB, 192(R26) + MOVV R23, 176(R26) + MOVV R24, 184(R26) + MOVV R25, 192(R26) + MOVV RSB, 200(R26) MOVV HI, R1 - MOVV R1, 200(R26) - MOVV LO, R1 MOVV R1, 208(R26) + MOVV LO, R1 + MOVV R1, 216(R26) RET // stolen from runtime/preempt_mips64x.s +// TODO why was temp register (R23) not included in preempt_mips64x? // R26 must point to stored gprs TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 - MOVV 208(R26), R1 + MOVV 216(R26), R1 MOVV R1, LO - MOVV 200(R26), R1 + MOVV 208(R26), R1 MOVV R1, HI - MOVV 192(R26), RSB - MOVV 184(R26), R25 - MOVV 176(R26), R24 + MOVV 200(R26), RSB + MOVV 192(R26), R25 + MOVV 184(R26), R24 + MOVV 176(R26), R23 MOVV 168(R26), R22 MOVV 160(R26), R21 MOVV 152(R26), R20 From da434ec9fb459480b951db5179d853bbc29bef9b Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 8 Nov 2023 12:39:08 +0100 Subject: [PATCH 19/45] Save and restore status register --- src/runtime/tasker_noos_mips64.go | 8 +++---- src/runtime/tasker_noos_mips64.s | 40 ++++++++++++------------------- 2 files changed, 19 insertions(+), 29 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index d573a5a536e457..c2b2d9866577a7 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -17,10 +17,10 @@ const ( ) type mOS struct { - gprs [numGPRS]uintptr - fprs [numFPRS]float64 - sp, fp uintptr // goroutine context - ra, epc uintptr // exception context + // thread context + gprs [numGPRS]uintptr + fprs [numFPRS]float64 + sp, fp, ra, epc uintptr } var bbplayer bool // TODO move to n64 board support package diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 4a04ca730a710d..e0f66630072f79 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -14,7 +14,7 @@ // Exception Context #define _LR (0*8) -#define _mcause (1*8) +#define _mstatus (1*8) #define _mepc (2*8) #define excCtxSize (3*8) @@ -83,8 +83,8 @@ fromHandler: SUB $excCtxSize, R29 OR $1, R31, R26 // encode smallCtx flag in ra MOVV R26, _LR(R29) - MOVV M(C0_CAUSE), R26 - MOVV R26, _mcause(R29) + MOVV M(C0_SR), R26 + MOVV R26, _mstatus(R29) MOVV M(C0_EPC), R26 OR R27, R26 // encode fromHandler flag in EPC MOVV R26, _mepc(R29) @@ -132,10 +132,11 @@ fatal: // R9: argument data size on the stack (+8 for caller return address) // R10: return data size on the stack TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 - // if fromHandler skip saving thread context MOVV _mepc(R29), R27 ADD $4, R27 // don't execute syscall instruction again MOVV R27, _mepc(R29) + + // if fromHandler skip saving thread context AND $1, R27 // fromHandler flag BNE R27, R0, currentStack @@ -216,11 +217,11 @@ nothingToCopy: BEQ R8, R0, 2(PC) JMP ·enterScheduler(SB) -// restore ctx of caller + // restore ctx of caller MOVV _LR(R29), R26 AND $~1, R26, R31 // remove smallCtx flag from ra -// MOVV _mcause(R29), R26 -// MOVV R26, M(C0_CAUSE) + MOVV _mstatus(R29), R26 + MOVV R26, M(C0_SR) MOVV _mepc(R29), R26 AND $1, R26, R27 AND $~1, R26 // remove fromHandler flag from epc @@ -228,11 +229,6 @@ nothingToCopy: ADD $excCtxSize, R29 - // unmask interrupts - MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_EXT), R26 - MOVV R26, M(C0_SR) - BNE R27, R0, return MOVV $·cpu0(SB), R26 @@ -279,9 +275,6 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 - // pop everthing from stack - ADD $excCtxSize, R29 - // reenable exceptions MOVV M(C0_SR), R26 AND $~SR_EXL, R26 @@ -301,6 +294,11 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 // clear cpuctx.newexe MOVB R0, (cpuctx_newexe)(g) + // restore mstatus from exception context + MOVV _mstatus(R29), R26 + MOVV R26, M(C0_SR) + ADD $excCtxSize, R29 + MOVV (cpuctx_exe)(g), R27 MOVV (m_mOS+mOS_ra)(R27), R9 AND $~1, R9, R23 // remove smallCtx flag @@ -313,11 +311,6 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 JAL ·restoreGPRs(SB) // TODO skip if we are coming from syscall smallCtx: - // unmask interrupts - MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_EXT), R26 - MOVV R26, M(C0_SR) - MOVV 0(R29), R26 ADD $8, R29 MOVV (m_mOS+mOS_sp)(R27), R29 @@ -392,16 +385,13 @@ callVector: OR $SR_EXL, R8 MOVV R8, M(C0_SR) - // Unmask interrupts - MOVV M(C0_SR), R26 - OR $(INTR_SW|INTR_EXT), R26 - MOVV R26, M(C0_SR) - // Restore ctx of caller MOVV R29, R26 JAL ·restoreGPRs(SB) ADD $const_numGPRS*8, R29 + MOVV _mstatus(R29), R26 + MOVV R26, M(C0_SR) MOVV _LR(R29), R26 MOVV $~1, R27 AND R27, R26, R31 // Remove smallCtx flag from RA From ef0642a361aea11946b1d0f181b1e541fc829583 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sat, 18 Nov 2023 19:29:12 +0100 Subject: [PATCH 20/45] Remove broken setAlarm() implementation --- .../arch/r4000/systim/systim_mips64.go | 36 ++++++++++++------- src/runtime/rt0_noos_mips64.s | 2 +- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/embedded/arch/r4000/systim/systim_mips64.go b/src/embedded/arch/r4000/systim/systim_mips64.go index 591b6fca3bed25..723a21d20662df 100644 --- a/src/embedded/arch/r4000/systim/systim_mips64.go +++ b/src/embedded/arch/r4000/systim/systim_mips64.go @@ -1,3 +1,9 @@ +// Current implementation has limitations: +// - Overflows at 2^56 nanoseconds (834 days) because 8 bits are used to +// improve precision of hz2ns variable. +// - Calls between nanotime must not be more than 2^32 nanoseconds apart, +// otherwise overflow detection fails. Since the GC calls nanoseconds +// frequently, it's currently not a issue. package systim import ( @@ -6,28 +12,34 @@ import ( _ "unsafe" // for linkname ) -var hz2ns int64 +var hz2ns int64 // 56.8 fixed-point integer +var lastTicks int64 -// Setup setups COUNT register to work as system timer. +// Setups COUNT register to work as system timer. func Setup(clkhz int64) { - hz2ns = 2 * 1e9 / clkhz - creg.COMPARE.Store(0xffffffff) + hz2ns = 2 * 1e9 + hz2ns = hz2ns << 8 + hz2ns /= clkhz + + setAlarm(nanotime()) // init lastTicks and alarm - creg.STATUS.SetBits(creg.IP_TIMER) - creg.CAUSE.ClearBits(creg.IP_TIMER) rtos.SetSystemTimer(nanotime, setAlarm) } //go:nosplit func nanotime() int64 { - return int64(creg.COUNT.Load()) * hz2ns + // TODO handle overflow in interrupt + ticks32 := int64(creg.COUNT.Load()) + if ticks32 < lastTicks&0xffff_ffff { + lastTicks += (1 << 32) + } + lastTicks = (lastTicks & ^0xffff_ffff) | ticks32 + + return (lastTicks * hz2ns) >> 8 } //go:nosplit func setAlarm(ns int64) { - var compare uint32 - if ns >= 0 { - compare = uint32(ns / hz2ns) - } - creg.COMPARE.Store(compare) + // Only needed for a tickless implementation. As long as curcpuSleep is + // a noop, there is no need to implement this. } diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 22d1883f50edfa..3baeef08b9b257 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -255,7 +255,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 // TODO where to enable interupts correctly? MOVW z0, M(C0_COMPARE) MOVW M(C0_SR), t0 - OR $(SR_IE|INTR_SW|INTR_TIMER), t0 + OR $(SR_IE|INTR_SW|INTR_EXT), t0 MOVW t0, M(C0_SR) // start this M From 8283719cbe8a475bacf7bd2214c5d9152bdb4673 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sat, 18 Nov 2023 19:54:09 +0100 Subject: [PATCH 21/45] Mitigate sign-extended pointers with TLB --- src/runtime/asm_mips64.h | 5 +++ src/runtime/const_noos_n64.go | 2 +- src/runtime/rt0_noos_mips64.s | 64 +++++++++++++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index 110e7995467f86..c2a2825841492e 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -67,7 +67,12 @@ #define PHYS_TO_K1(a) ((unsigned)(a) | K1BASE) /* Standard Co-Processor 0 registers */ +#define C0_INDEX 0 /* Index of TLB Entry */ +#define C0_ENTRYLO0 2 /* TLB entry's first PFN */ +#define C0_ENTRYLO1 3 /* TLB entry's second PFN */ +#define C0_PAGEMASK 5 /* Size of TLB Entries */ #define C0_COUNT 9 /* Timer Count Register */ +#define C0_ENTRYHI 10 /* VPN and ASID of two TLB entry */ #define C0_COMPARE 11 /* Timer Compare Register */ #define C0_SR 12 /* Status Register */ #define C0_CAUSE 13 /* last exception description */ diff --git a/src/runtime/const_noos_n64.go b/src/runtime/const_noos_n64.go index e9437e889938f7..fede0e0af49d10 100644 --- a/src/runtime/const_noos_n64.go +++ b/src/runtime/const_noos_n64.go @@ -17,7 +17,7 @@ const ( noosNumStackOrders = 3 noosHeapAddrBits = 23 // enough for 8 MiB K210 SRAM noosLogHeapArenaBytes = 17 // 128 KiB - noosArenaBaseOffset = 0xffffffff80000000 + noosArenaBaseOffset = 0x0 noosMinPhysPageSize = 256 noosSpanSetInitSpineCap = 64 noosStackMin = 2048 diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 3baeef08b9b257..08082d5fad1fa7 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -10,12 +10,16 @@ #define DEFAULT_C0_SR SR_CU1|SR_PE|SR_FR +#define TLBWI WORD $0x42000002 + TEXT _rt0_mips64_noos(SB),NOSPLIT|NOFRAME,$0 // Watchpoints have been proven to persist across resets and even with // the console being off. Zero it as early as possible, to avoid it // triggering during boot. This should really be done at the start IPL3. MOVW z0, M(C0_WATCHLO) + JAL runtime·rt0_tlb(SB) + // Check whether we are running on iQue or N64. Use the MI version // register which has LSB set to 0xB0 on iQue. We assume 0xBn was meant // for BBPlayer. Notice that we want this test to be hard for emulators @@ -40,8 +44,8 @@ TEXT _rt0_mips64_noos(SB),NOSPLIT|NOFRAME,$0 MOVW 0x7C0000, t0 set_sp: - MOVW $0x7FFFFFF0, t1 - ADDU t0,t1,sp // init stack pointer + MOVV $0x10, t1 + SUBV t1, t0, sp // init stack pointer MOVV $0, RSB // init data pointer MOVW $8, v0 MOVW v0, (0xbfc007fc) // magic N64 hardware init @@ -54,6 +58,7 @@ set_sp: // Check if PI DMA transfer is required, knowing that IPL3 loads 1 MiB // of ROM to RAM, and __libdragon_text_start is located right after the // ROM header where this 1 MiB starts. + // TODO test again with TLB mappings configured MOVW $_rt0_mips64_noos(SB), a0 MOVW $runtime·edata(SB), a1 MOVW $0x100000, t0 // stock IPL3 load size (1 MiB) @@ -141,6 +146,61 @@ loadintvectorloop: JMP runtime·rt0_go(SB) +// Maps KSEG0, KSEG1 and ROM to the beginning of the virtual address space. +// This saves us from sign-extending pointers correctly, as we avoid pointers +// upwards of 0x80000000. +// The only problems encountered with falsely sign-extended pointers were symbol +// addresses loaded from the ELF. Otherwise running code in KSEG0 seems to be +// no problem in general. +// The correct way of doing this would probably involve defining a new 64-bit +// architecture with PtrSize = 4, but I have ran into some issues that weren't +// worth fixing at the moment. +// TODO currently only 16 MiB of cartridge is mapped +TEXT runtime·rt0_tlb(SB),NOSPLIT|NOFRAME,$0 + MOVV $0, R8 + MOVV R8, M(C0_INDEX) + MOVV $0xfff << 13, R8 + MOVV R8, M(C0_PAGEMASK) + MOVV $(0x00000000 >> 6) | 0x7, R8 + MOVV R8, M(C0_ENTRYLO0) + MOVV $(0x01000000 >> 6) | 0x7, R8 + MOVV R8, M(C0_ENTRYLO1) + MOVV $0x00000000, R8 + MOVV R8, M(C0_ENTRYHI) + NOOP // avert CP0 hazards + TLBWI + + MOVV $1, R8 + MOVV R8, M(C0_INDEX) + MOVV $0xfff << 13, R8 + MOVV R8, M(C0_PAGEMASK) + MOVV $(0x10000000 >> 6) | (2<<3) | 0x3, R8 + MOVV R8, M(C0_ENTRYLO0) + MOVV $(0x11000000 >> 6) | (2<<3) | 0x3, R8 + MOVV R8, M(C0_ENTRYLO1) + MOVV $0x10000000, R8 + MOVV R8, M(C0_ENTRYHI) + NOOP // avert CP0 hazards + TLBWI + + MOVV $2, R8 + MOVV R8, M(C0_INDEX) + MOVV $0xfff << 13, R8 + MOVV R8, M(C0_PAGEMASK) + MOVV $(0x00000000 >> 6) | (2<<3) | 0x7, R8 + MOVV R8, M(C0_ENTRYLO0) + MOVV $(0x01000000 >> 6) | (2<<3) | 0x7, R8 + MOVV R8, M(C0_ENTRYLO1) + MOVV $0x20000000, R8 + MOVV R8, M(C0_ENTRYHI) + NOOP // avert CP0 hazards + TLBWI + + MOVV $0x7fffffff, R8 + AND R8, ra // return to the tlb mapped address + RET + + #define PALLOC_MIN 20*1024 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 From 6f008408de717a0f925ec181db5f9d22897a2130 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sat, 18 Nov 2023 20:11:03 +0100 Subject: [PATCH 22/45] Don't allow preempting the scheduler for now --- src/runtime/tasker_noos_mips64.go | 9 ++++++++- src/runtime/tasker_noos_mips64.s | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index c2b2d9866577a7..deba503fb126b8 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -45,7 +45,14 @@ func taskerinit() { // nothing. // //go:nosplit -func curcpuSleep() {} +func curcpuSleep() { + // Check defensively for external interrupts here. There were lockups + // when allowing the scheduler to be interrupted. + // TODO try again, this might have been my own fault by breaking + // atomic operations. + creg.STATUS.SetBits(creg.IP_EXT) + creg.STATUS.ClearBits(creg.IP_EXT) +} // This function is used to inform the another CPU that there is a new thread // added to its runnable queue. It should wake up the sleeping CPU or preempt diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index e0f66630072f79..06a8e0081de7b6 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -278,6 +278,7 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 // reenable exceptions MOVV M(C0_SR), R26 AND $~SR_EXL, R26 +// OR $INTR_EXT, R26 // allow external interrupts to preempt scheduler MOVV R26, M(C0_SR) // reminder: don't use R26 or R27 when interrupts enabled From 430c3cbb06549e477e0b79380d269b0924a31713 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sat, 18 Nov 2023 23:38:08 +0100 Subject: [PATCH 23/45] Add makeshift USB logging --- src/runtime/asm_mips64.h | 1 + src/runtime/rt0_noos_mips64.s | 2 + src/runtime/tasker_noos_mips64.go | 60 +++++++-- src/runtime/tasker_noos_mips64.s | 213 +++++++++++++++++++++++++++++- 4 files changed, 265 insertions(+), 11 deletions(-) diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index c2a2825841492e..08d036981f0859 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -71,6 +71,7 @@ #define C0_ENTRYLO0 2 /* TLB entry's first PFN */ #define C0_ENTRYLO1 3 /* TLB entry's second PFN */ #define C0_PAGEMASK 5 /* Size of TLB Entries */ +#define C0_BADVADDR 8 /* Address that caused exception */ #define C0_COUNT 9 /* Timer Count Register */ #define C0_ENTRYHI 10 /* VPN and ASID of two TLB entry */ #define C0_COMPARE 11 /* Timer Compare Register */ diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 08082d5fad1fa7..d9c1023867cf5b 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -143,6 +143,8 @@ loadintvectorloop: ADDU $-1, t2 BGTZ t2,loadintvectorloop + JAL ·initEverdrive64USB(SB) // get ready for logging + JMP runtime·rt0_go(SB) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index deba503fb126b8..51c30bbc12bbdf 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -128,25 +128,65 @@ func restoreGPRs() func saveFPRs() func restoreFPRs() -var lastlog [1024 * 16]byte -var lastidx = 0 +var usbBuffer [512 + 16]byte // pad by 16 byte for aligment and cache ops -// Just write into some ringbuffer for now, which can be inspected in a memory -// dump. +//go:nosplit +func printFatal(cause uint, epc uint, status uint, badvaddr uint, ra uint) { + // don't use print() here since it will cause a syscall exception + defaultWrite(0, []byte("--- FATAL exception ---\n")) + defaultWrite(0, []byte("cause: 0x")) + printUint32Hex(cause) + defaultWrite(0, []byte("\nepc: 0x")) + printUint32Hex(epc) + defaultWrite(0, []byte("\nstatus: 0x")) + printUint32Hex(status) + defaultWrite(0, []byte("\nbadvaddr: 0x")) + printUint32Hex(badvaddr) + defaultWrite(0, []byte("\nra: 0x")) + printUint32Hex(ra) + defaultWrite(0, []byte("\n--------- END ---------\n")) +} + +// Prints a uint32 in hex format. +// +//go:nosplit +func printUint32Hex(n uint) { + var buf [8]byte + for i := range buf { + ascii := byte(n>>((7-i)*4)) & 0xf + ascii += 48 + if ascii > 57 { + ascii += 39 + } + buf[i] = ascii + } + defaultWrite(0, buf[:]) +} + +// Sends text messages over USB compatible with UNFLoader. // //go:nowritebarrierrec //go:nosplit func defaultWrite(fd int, p []byte) int { - for i := 0; i < len(p); i++ { - lastlog[lastidx] = p[i] - lastidx += 1 - if lastidx >= len(lastlog) { - lastidx = 0 - } + alignedStart := 16 - int(uintptr(unsafe.Pointer(&usbBuffer))%16) + alignedEnd := alignedStart + len(p) + 12 + if alignedEnd%2 == 1 { + alignedEnd += 1 } + usbBufferAligned := usbBuffer[alignedStart:alignedEnd] + + copy(usbBufferAligned, "DMA@") + copy(usbBufferAligned[4:], []byte{1, 0, 0, byte(len(p))}) + copy(usbBufferAligned[8:], p) + copy(usbBufferAligned[8+len(p):], "CMPH") + ptr := unsafe.SliceData(usbBufferAligned) + writeEverdrive64USB(unsafe.Pointer(ptr), len(usbBufferAligned)) return len(p) } +func initEverdrive64USB() +func writeEverdrive64USB(p unsafe.Pointer, size int) + // syscalls not used by runtime func syssetprivlevel(newlevel int) (oldlevel, errno int) { return } // TODO func sysirqctl(irq, ctl, ctxid int) (enabled, prio, errno int) { return } // TODO diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 06a8e0081de7b6..f3370946d9a64f 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -119,7 +119,18 @@ fromHandler: fatal: // unhandled exception - BREAK + MOVV M(C0_CAUSE), R26 + MOVV R26, 8(R29) + MOVV M(C0_EPC), R26 + MOVV R26, 16(R29) + MOVV M(C0_SR), R26 + MOVV R26, 24(R29) + MOVV M(C0_BADVADDR), R26 + MOVV R26, 32(R29) + MOVV R31, 40(R29) + JAL ·printFatal(SB) + + NOP JMP -1(PC) @@ -594,3 +605,203 @@ TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 // func syscachemaint(op int, p unsafe.Pointer, size int) TEXT ·syscachemaint(SB),NOSPLIT,$0-12 NOP // TODO + + +// Initializes EverDrive64 USB and sends a single heartbeat packet to tell +// UNFLoader we use protocol V2 with 2 byte alignment. Only tested with +// EverDrive64 X7. +// TODO support other carts (drive64, SummerCart, PicoCart) +TEXT runtime·initEverdrive64USB(SB),NOSPLIT|NOFRAME,$0 + MOVW $0xBF808004, R26 + MOVW $0xAA55, R27 + MOVW R27, (R26) // unlock ED64 registers + + MOVW $0xA4600000, t0 +wait_unlock_end: + MOVW 0x10(t0), t1 // PI_STATUS + AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ t1, wait_unlock_end + +loop: + MOVW $0xBF800004, R26 + MOVW $0xC000, R27 // WRNOP + MOVW R27, (R26) + + MOVW $0xA4600000, t0 +wait_wrnop: + MOVW 0x10(t0), t1 // PI_STATUS + AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ t1, wait_wrnop + + MOVW $0x80400400+512-16, R26 + MOVW $68, R27 // 'D' + MOVB R27, (R26) + MOVW $0x80400400+512-15, R26 + MOVW $77, R27 // 'M' + MOVB R27, (R26) + MOVW $0x80400400+512-14, R26 + MOVW $65, R27 // 'A' + MOVB R27, (R26) + MOVW $0x80400400+512-13, R26 + MOVW $64, R27 // '@' + MOVB R27, (R26) + MOVW $0x80400400+512-12, R26 + MOVW $5, R27 // type: heartbeat + MOVB R27, (R26) + MOVW $0x80400400+512-11, R26 + MOVW $0, R27 + MOVB R27, (R26) + MOVW $0x80400400+512-10, R26 + MOVW $0, R27 + MOVB R27, (R26) + MOVW $0x80400400+512-9, R26 + MOVW $4, R27 + MOVB R27, (R26) + MOVW $0x80400400+512-8, R26 + MOVW $0, R27 + MOVB R27, (R26) + MOVW $0x80400400+512-7, R26 + MOVW $2, R27 + MOVB R27, (R26) + MOVW $0x80400400+512-6, R26 + MOVW $0, R27 + MOVB R27, (R26) + MOVW $0x80400400+512-5, R26 + MOVW $1, R27 + MOVB R27, (R26) + MOVW $0x80400400+512-4, R26 + MOVW $67, R27 // 'C' + MOVB R27, (R26) + MOVW $0x80400400+512-3, R26 + MOVW $77, R27 // 'M' + MOVB R27, (R26) + MOVW $0x80400400+512-2, R26 + MOVW $80, R27 // 'P' + MOVB R27, (R26) + MOVW $0x80400400+512-1, R26 + MOVW $72, R27 // 'H' + MOVB R27, (R26) + + // invalidate cache + MOVW $0x80400400+512, R26 + BREAK R25, 0(R26) + BREAK R25, -16(R26) + + // Start PI DMA transfer + MOVW $0x80400400+512-16, a0 + MOVW $0x1F800400+512-16, a1 + MOVW $16, a2 + MOVW $0xA4600000, t0 + MOVW a0, 0x00(t0) // PI_DRAM_ADDR + MOVW a1, 0x04(t0) // PI_CART_ADDR + MOVW a2, 0x08(t0) // PI_RD_LEN + +wait_dma_end: + MOVW 0x10(t0), t1 // PI_STATUS + AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ t1, wait_dma_end + + MOVW $0xBF800004, R26 + MOVW $0xC200+512-16, R27 // WR + baseaddr + MOVW R27, (R26) + + MOVW $0xA4600000, t0 +wait_wr: + MOVW 0x10(t0), t1 // PI_STATUS + AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ t1, wait_wr + +wait: + MOVW $0xBF800004, R26 + MOVW (R26), R27 + AND $0x200, R27 + BNE R0, R27, wait + + RET + + +// p must be 16 byte aligned. Size must be less-or-equal 512 byte. +// TODO implement a function to safely write PI registers instead of waiting for +// IO_BUSY after each write. Also use it in initEverdrive64USB. +// +// func writeEverdrive64USB(p unsafe.Pointer, size int) +TEXT runtime·writeEverdrive64USB(SB),NOSPLIT|NOFRAME,$0-16 + MOVV p+0(FP), R4 + MOVV size+8(FP), R5 + + MOVW $0xBF800004, R8 + MOVW $0xC000, R9 // WRNOP + MOVW R9, (R8) + + MOVW $0xA4600000, R8 +wait_wrnop: + MOVW 0x10(R8), R9 // PI_STATUS + AND $3, R9 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ R9, wait_wrnop + + // writeback cache + // TODO loop over actual length + BREAK R25, 0(R4) + BREAK R25, 16(R4) + BREAK R25, 32(R4) + BREAK R25, 48(R4) + BREAK R25, 64(R4) + BREAK R25, 80(R4) + BREAK R25, 96(R4) + BREAK R25, 112(R4) + BREAK R25, 128(R4) + BREAK R25, 144(R4) + BREAK R25, 160(R4) + BREAK R25, 176(R4) + BREAK R25, 192(R4) + BREAK R25, 208(R4) + BREAK R25, 224(R4) + BREAK R25, 240(R4) + BREAK R25, 256(R4) + BREAK R25, 272(R4) + BREAK R25, 288(R4) + BREAK R25, 304(R4) + BREAK R25, 320(R4) + BREAK R25, 336(R4) + BREAK R25, 352(R4) + BREAK R25, 368(R4) + BREAK R25, 384(R4) + BREAK R25, 400(R4) + BREAK R25, 416(R4) + BREAK R25, 432(R4) + BREAK R25, 448(R4) + BREAK R25, 464(R4) + BREAK R25, 480(R4) + BREAK R25, 496(R4) + + // Start PI DMA transfer + MOVW $0x1F800400+512, R10 + SUB R5, R10 + MOVW $0xA4600000, R8 + MOVW R4, 0x00(R8) // PI_DRAM_ADDR + MOVW R10, 0x04(R8) // PI_CART_ADDR + MOVW R5, 0x08(R8) // PI_RD_LEN + +wait_dma_end: + MOVW 0x10(R8), R9 // PI_STATUS + AND $3, R9 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ R9, wait_dma_end + + MOVW $0xBF800004, R8 + MOVW $0xC200+512, R9 // WR + baseaddr + SUB R5, R9 + MOVW R9, (R8) + + MOVW $0xA4600000, R8 +wait_wr: + MOVW 0x10(R8), R9 // PI_STATUS + AND $3, R9 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY + BGTZ R9, wait_wr + +wait: + MOVW $0xBF800004, R8 + MOVW (R8), R9 + AND $0x200, R9 + BNE R0, R9, wait + + RET From 298a74f91198f649ddce8d45e84f4c47ea27e671 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Tue, 11 Jun 2024 15:33:09 +0200 Subject: [PATCH 24/45] Revert "Add makeshift USB logging" This reverts commit a292acb81065410b72259cf5e9c21ca12370df13. --- src/runtime/asm_mips64.h | 1 - src/runtime/rt0_noos_mips64.s | 2 - src/runtime/tasker_noos_mips64.go | 60 ++------- src/runtime/tasker_noos_mips64.s | 213 +----------------------------- 4 files changed, 11 insertions(+), 265 deletions(-) diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index 08d036981f0859..c2a2825841492e 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -71,7 +71,6 @@ #define C0_ENTRYLO0 2 /* TLB entry's first PFN */ #define C0_ENTRYLO1 3 /* TLB entry's second PFN */ #define C0_PAGEMASK 5 /* Size of TLB Entries */ -#define C0_BADVADDR 8 /* Address that caused exception */ #define C0_COUNT 9 /* Timer Count Register */ #define C0_ENTRYHI 10 /* VPN and ASID of two TLB entry */ #define C0_COMPARE 11 /* Timer Compare Register */ diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index d9c1023867cf5b..08082d5fad1fa7 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -143,8 +143,6 @@ loadintvectorloop: ADDU $-1, t2 BGTZ t2,loadintvectorloop - JAL ·initEverdrive64USB(SB) // get ready for logging - JMP runtime·rt0_go(SB) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index 51c30bbc12bbdf..deba503fb126b8 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -128,65 +128,25 @@ func restoreGPRs() func saveFPRs() func restoreFPRs() -var usbBuffer [512 + 16]byte // pad by 16 byte for aligment and cache ops +var lastlog [1024 * 16]byte +var lastidx = 0 -//go:nosplit -func printFatal(cause uint, epc uint, status uint, badvaddr uint, ra uint) { - // don't use print() here since it will cause a syscall exception - defaultWrite(0, []byte("--- FATAL exception ---\n")) - defaultWrite(0, []byte("cause: 0x")) - printUint32Hex(cause) - defaultWrite(0, []byte("\nepc: 0x")) - printUint32Hex(epc) - defaultWrite(0, []byte("\nstatus: 0x")) - printUint32Hex(status) - defaultWrite(0, []byte("\nbadvaddr: 0x")) - printUint32Hex(badvaddr) - defaultWrite(0, []byte("\nra: 0x")) - printUint32Hex(ra) - defaultWrite(0, []byte("\n--------- END ---------\n")) -} - -// Prints a uint32 in hex format. -// -//go:nosplit -func printUint32Hex(n uint) { - var buf [8]byte - for i := range buf { - ascii := byte(n>>((7-i)*4)) & 0xf - ascii += 48 - if ascii > 57 { - ascii += 39 - } - buf[i] = ascii - } - defaultWrite(0, buf[:]) -} - -// Sends text messages over USB compatible with UNFLoader. +// Just write into some ringbuffer for now, which can be inspected in a memory +// dump. // //go:nowritebarrierrec //go:nosplit func defaultWrite(fd int, p []byte) int { - alignedStart := 16 - int(uintptr(unsafe.Pointer(&usbBuffer))%16) - alignedEnd := alignedStart + len(p) + 12 - if alignedEnd%2 == 1 { - alignedEnd += 1 + for i := 0; i < len(p); i++ { + lastlog[lastidx] = p[i] + lastidx += 1 + if lastidx >= len(lastlog) { + lastidx = 0 + } } - usbBufferAligned := usbBuffer[alignedStart:alignedEnd] - - copy(usbBufferAligned, "DMA@") - copy(usbBufferAligned[4:], []byte{1, 0, 0, byte(len(p))}) - copy(usbBufferAligned[8:], p) - copy(usbBufferAligned[8+len(p):], "CMPH") - ptr := unsafe.SliceData(usbBufferAligned) - writeEverdrive64USB(unsafe.Pointer(ptr), len(usbBufferAligned)) return len(p) } -func initEverdrive64USB() -func writeEverdrive64USB(p unsafe.Pointer, size int) - // syscalls not used by runtime func syssetprivlevel(newlevel int) (oldlevel, errno int) { return } // TODO func sysirqctl(irq, ctl, ctxid int) (enabled, prio, errno int) { return } // TODO diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index f3370946d9a64f..06a8e0081de7b6 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -119,18 +119,7 @@ fromHandler: fatal: // unhandled exception - MOVV M(C0_CAUSE), R26 - MOVV R26, 8(R29) - MOVV M(C0_EPC), R26 - MOVV R26, 16(R29) - MOVV M(C0_SR), R26 - MOVV R26, 24(R29) - MOVV M(C0_BADVADDR), R26 - MOVV R26, 32(R29) - MOVV R31, 40(R29) - JAL ·printFatal(SB) - - NOP + BREAK JMP -1(PC) @@ -605,203 +594,3 @@ TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 // func syscachemaint(op int, p unsafe.Pointer, size int) TEXT ·syscachemaint(SB),NOSPLIT,$0-12 NOP // TODO - - -// Initializes EverDrive64 USB and sends a single heartbeat packet to tell -// UNFLoader we use protocol V2 with 2 byte alignment. Only tested with -// EverDrive64 X7. -// TODO support other carts (drive64, SummerCart, PicoCart) -TEXT runtime·initEverdrive64USB(SB),NOSPLIT|NOFRAME,$0 - MOVW $0xBF808004, R26 - MOVW $0xAA55, R27 - MOVW R27, (R26) // unlock ED64 registers - - MOVW $0xA4600000, t0 -wait_unlock_end: - MOVW 0x10(t0), t1 // PI_STATUS - AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ t1, wait_unlock_end - -loop: - MOVW $0xBF800004, R26 - MOVW $0xC000, R27 // WRNOP - MOVW R27, (R26) - - MOVW $0xA4600000, t0 -wait_wrnop: - MOVW 0x10(t0), t1 // PI_STATUS - AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ t1, wait_wrnop - - MOVW $0x80400400+512-16, R26 - MOVW $68, R27 // 'D' - MOVB R27, (R26) - MOVW $0x80400400+512-15, R26 - MOVW $77, R27 // 'M' - MOVB R27, (R26) - MOVW $0x80400400+512-14, R26 - MOVW $65, R27 // 'A' - MOVB R27, (R26) - MOVW $0x80400400+512-13, R26 - MOVW $64, R27 // '@' - MOVB R27, (R26) - MOVW $0x80400400+512-12, R26 - MOVW $5, R27 // type: heartbeat - MOVB R27, (R26) - MOVW $0x80400400+512-11, R26 - MOVW $0, R27 - MOVB R27, (R26) - MOVW $0x80400400+512-10, R26 - MOVW $0, R27 - MOVB R27, (R26) - MOVW $0x80400400+512-9, R26 - MOVW $4, R27 - MOVB R27, (R26) - MOVW $0x80400400+512-8, R26 - MOVW $0, R27 - MOVB R27, (R26) - MOVW $0x80400400+512-7, R26 - MOVW $2, R27 - MOVB R27, (R26) - MOVW $0x80400400+512-6, R26 - MOVW $0, R27 - MOVB R27, (R26) - MOVW $0x80400400+512-5, R26 - MOVW $1, R27 - MOVB R27, (R26) - MOVW $0x80400400+512-4, R26 - MOVW $67, R27 // 'C' - MOVB R27, (R26) - MOVW $0x80400400+512-3, R26 - MOVW $77, R27 // 'M' - MOVB R27, (R26) - MOVW $0x80400400+512-2, R26 - MOVW $80, R27 // 'P' - MOVB R27, (R26) - MOVW $0x80400400+512-1, R26 - MOVW $72, R27 // 'H' - MOVB R27, (R26) - - // invalidate cache - MOVW $0x80400400+512, R26 - BREAK R25, 0(R26) - BREAK R25, -16(R26) - - // Start PI DMA transfer - MOVW $0x80400400+512-16, a0 - MOVW $0x1F800400+512-16, a1 - MOVW $16, a2 - MOVW $0xA4600000, t0 - MOVW a0, 0x00(t0) // PI_DRAM_ADDR - MOVW a1, 0x04(t0) // PI_CART_ADDR - MOVW a2, 0x08(t0) // PI_RD_LEN - -wait_dma_end: - MOVW 0x10(t0), t1 // PI_STATUS - AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ t1, wait_dma_end - - MOVW $0xBF800004, R26 - MOVW $0xC200+512-16, R27 // WR + baseaddr - MOVW R27, (R26) - - MOVW $0xA4600000, t0 -wait_wr: - MOVW 0x10(t0), t1 // PI_STATUS - AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ t1, wait_wr - -wait: - MOVW $0xBF800004, R26 - MOVW (R26), R27 - AND $0x200, R27 - BNE R0, R27, wait - - RET - - -// p must be 16 byte aligned. Size must be less-or-equal 512 byte. -// TODO implement a function to safely write PI registers instead of waiting for -// IO_BUSY after each write. Also use it in initEverdrive64USB. -// -// func writeEverdrive64USB(p unsafe.Pointer, size int) -TEXT runtime·writeEverdrive64USB(SB),NOSPLIT|NOFRAME,$0-16 - MOVV p+0(FP), R4 - MOVV size+8(FP), R5 - - MOVW $0xBF800004, R8 - MOVW $0xC000, R9 // WRNOP - MOVW R9, (R8) - - MOVW $0xA4600000, R8 -wait_wrnop: - MOVW 0x10(R8), R9 // PI_STATUS - AND $3, R9 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ R9, wait_wrnop - - // writeback cache - // TODO loop over actual length - BREAK R25, 0(R4) - BREAK R25, 16(R4) - BREAK R25, 32(R4) - BREAK R25, 48(R4) - BREAK R25, 64(R4) - BREAK R25, 80(R4) - BREAK R25, 96(R4) - BREAK R25, 112(R4) - BREAK R25, 128(R4) - BREAK R25, 144(R4) - BREAK R25, 160(R4) - BREAK R25, 176(R4) - BREAK R25, 192(R4) - BREAK R25, 208(R4) - BREAK R25, 224(R4) - BREAK R25, 240(R4) - BREAK R25, 256(R4) - BREAK R25, 272(R4) - BREAK R25, 288(R4) - BREAK R25, 304(R4) - BREAK R25, 320(R4) - BREAK R25, 336(R4) - BREAK R25, 352(R4) - BREAK R25, 368(R4) - BREAK R25, 384(R4) - BREAK R25, 400(R4) - BREAK R25, 416(R4) - BREAK R25, 432(R4) - BREAK R25, 448(R4) - BREAK R25, 464(R4) - BREAK R25, 480(R4) - BREAK R25, 496(R4) - - // Start PI DMA transfer - MOVW $0x1F800400+512, R10 - SUB R5, R10 - MOVW $0xA4600000, R8 - MOVW R4, 0x00(R8) // PI_DRAM_ADDR - MOVW R10, 0x04(R8) // PI_CART_ADDR - MOVW R5, 0x08(R8) // PI_RD_LEN - -wait_dma_end: - MOVW 0x10(R8), R9 // PI_STATUS - AND $3, R9 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ R9, wait_dma_end - - MOVW $0xBF800004, R8 - MOVW $0xC200+512, R9 // WR + baseaddr - SUB R5, R9 - MOVW R9, (R8) - - MOVW $0xA4600000, R8 -wait_wr: - MOVW 0x10(R8), R9 // PI_STATUS - AND $3, R9 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ R9, wait_wr - -wait: - MOVW $0xBF800004, R8 - MOVW (R8), R9 - AND $0x200, R9 - BNE R0, R9, wait - - RET From 43723829de20905538df9b3125a916f657c18ceb Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Sat, 18 Nov 2023 23:41:03 +0100 Subject: [PATCH 25/45] Fix CP0 hazards of ERET --- src/runtime/tasker_noos_mips64.s | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 06a8e0081de7b6..09fbd6ee4c29eb 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -236,6 +236,8 @@ nothingToCopy: MOVV (g_sched+gobuf_g)(R26), g return: + NOOP // avert CP0 hazards + NOOP ERET @@ -320,6 +322,8 @@ smallCtx: MOVV (m_mOS+mOS_epc)(R27), R26 MOVV R26, M(C0_EPC) + NOOP // avert CP0 hazards + NOOP ERET @@ -414,6 +418,8 @@ callVector: MOVV (g_sched+gobuf_g)(R26), g return: + NOOP // avert CP0 hazards + NOOP ERET fatal: From 9fc8ef8400c75fdbfd129b2903d001a34d088953 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 22 May 2024 15:50:45 +0200 Subject: [PATCH 26/45] Declare interrupt handler in go files --- src/runtime/tasker_noos_mips64.go | 2 ++ src/runtime/tasker_noos_mips64.s | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index deba503fb126b8..05750dd407a312 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -147,6 +147,8 @@ func defaultWrite(fd int, p []byte) int { return len(p) } +func unhandledExternalInterrupt() + // syscalls not used by runtime func syssetprivlevel(newlevel int) (oldlevel, errno int) { return } // TODO func sysirqctl(irq, ctl, ctxid int) (enabled, prio, errno int) { return } // TODO diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 09fbd6ee4c29eb..8fe8431130900f 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -427,7 +427,7 @@ fatal: // Required by the linker, runtime.vectors will default to this. -TEXT runtime·unhandledExternalInterrupt(SB),NOSPLIT|NOFRAME,$0 +TEXT ·unhandledExternalInterrupt(SB),NOSPLIT|NOFRAME,$0 BREAK JMP -1(PC) From 4fd5ff8590c983f95167367541d494984248824b Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 22 May 2024 16:42:59 +0200 Subject: [PATCH 27/45] Fix instruction for accessing boolean variable --- src/runtime/rt0_noos_mips64.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 08082d5fad1fa7..980a0804547f19 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -120,7 +120,7 @@ wait_dma_end: BGTZ t1, wait_dma_end // Store the bbplayer flag now that BSS has been cleared - MOVW t7, runtime·bbplayer(SB) + MOVB t7, runtime·bbplayer(SB) // load interrupt vector MOVW $·intvector(SB), t0 From fd80e6d4491ab67fc7e87c70aef8c6b6cfe4a328 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Mon, 27 May 2024 11:02:06 +0200 Subject: [PATCH 28/45] Fix error message strings --- src/syscall/syscall_noos.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/syscall/syscall_noos.go b/src/syscall/syscall_noos.go index cb6942372ff78c..294e71cf71ecbd 100644 --- a/src/syscall/syscall_noos.go +++ b/src/syscall/syscall_noos.go @@ -19,9 +19,9 @@ type Error struct{ s string } var ( ENOTSUP = &Error{"operation not supported"} - EINVAL = &Error{"operation not supported"} - ENOENT = &Error{"invalid argument"} - EACCES = &Error{"no such file or directory"} + EINVAL = &Error{"invalid argument"} + ENOENT = &Error{"no such file or directory"} + EACCES = &Error{"permission denied"} EPERM = &Error{"operation not permitted"} EEXIST = &Error{"file exists"} ENOTEMPTY = &Error{"directory not empty"} @@ -33,7 +33,7 @@ var ( ENOSPC = &Error{"no space left on device"} EBADF = &Error{"bad file descriptor"} ECANCELED = &Error{"operation canceled"} - EINTR = &Error{"interrupt"} // for os package only + EINTR = &Error{"interrupt"} // for os package only //ENOPROTOOPT = &Error{"protocol not available"} // for net package only ) From 704da875044d59e41fb98f6557c3d1825b869e0b Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Mon, 10 Jun 2024 09:00:37 +0200 Subject: [PATCH 29/45] Increase space for non-heap allocations --- src/runtime/rt0_noos_mips64.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 980a0804547f19..6bfe96e01e6ced 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -233,7 +233,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 SUB t0, t1, t5 // size of available memory (DMA capable) // estimate the space need for non-heap allocations - SRL $(const__PageShift+1), t5, t4 + SRL $(const__PageShift), t5, t4 MOVV $mspan__size, t2 MUL t2, t4 MOVV LO, t4 From ac7609edc97bf52f422ee952d620448c4554386f Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 12 Jun 2024 23:30:25 +0200 Subject: [PATCH 30/45] Make sure dummyNanotime doesn't block in usleep() This mainly prevented to print panics before the systimer was set up. See the usleep() calls in freezetheworld(). This comes with the possibility to have a non-monotonic jump in time when the systimer is finally set. Systimer implementations must respect the dummyNanoseconds if there is a risk. --- src/runtime/tasker_noos.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/runtime/tasker_noos.go b/src/runtime/tasker_noos.go index 1cf23fa93efb0f..a835d3c4f26506 100644 --- a/src/runtime/tasker_noos.go +++ b/src/runtime/tasker_noos.go @@ -75,8 +75,10 @@ import ( // Tasker code does not use FPU so the architecture specific context switch // code can avoid saving/restoring FPU context if not need. +var dummyNanoseconds int64 + //go:nosplit -func dummyNanotime() int64 { return 1 } +func dummyNanotime() int64 { dummyNanoseconds += 100; return dummyNanoseconds } //go:nosplit func dummySetalarm(ns int64) {} From 9e816c3fd8b34119be66adbbbe5583b9094fb901 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 13 Jun 2024 00:43:14 +0200 Subject: [PATCH 31/45] Add dependency to target specific machine package --- src/runtime/rt0_noos_mips64.s | 220 ++++++------------------------ src/runtime/target_noos_n64.go | 11 ++ src/runtime/target_noos_n64.s | 7 + src/runtime/tasker_noos_mips64.go | 28 +--- 4 files changed, 67 insertions(+), 199 deletions(-) create mode 100644 src/runtime/target_noos_n64.go create mode 100644 src/runtime/target_noos_n64.s diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 6bfe96e01e6ced..93224edf5a3ee9 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -8,199 +8,65 @@ #include "textflag.h" #include "asm_mips64.h" -#define DEFAULT_C0_SR SR_CU1|SR_PE|SR_FR - -#define TLBWI WORD $0x42000002 - TEXT _rt0_mips64_noos(SB),NOSPLIT|NOFRAME,$0 - // Watchpoints have been proven to persist across resets and even with - // the console being off. Zero it as early as possible, to avoid it - // triggering during boot. This should really be done at the start IPL3. - MOVW z0, M(C0_WATCHLO) - - JAL runtime·rt0_tlb(SB) - - // Check whether we are running on iQue or N64. Use the MI version - // register which has LSB set to 0xB0 on iQue. We assume 0xBn was meant - // for BBPlayer. Notice that we want this test to be hard for emulators - // to pass by mistake, so checking for a specific value while reading - // seems solid enough. - MOVW (0x80000318), t0 // memory size - MOVW (0xA4300004), t1 - MOVW $0xB0, t2 - AND $0xF0, t1 - MOVW $0, t7 // t7=0 -> vanilla N64 - BNE t1, t2, set_sp - - // In iQue player, memory allocated to game can be configured and it - // appears in 0x80000318. On the other hand, the top 8 MiB of RDRAM is - // reserved to savegames. So avoid putting the stack there, capping the - // size to 0x7C0000. See also get_memory_size. - MOVW $1, t7 // t7=1 -> iQue player - MOVW 0x800000, t1 - SGT t2, t1, t0 - MOVW 0x0, t1 - BNE t2, t1, set_sp - MOVW 0x7C0000, t0 - -set_sp: - MOVV $0x10, t1 - SUBV t1, t0, sp // init stack pointer - MOVV $0, RSB // init data pointer - MOVW $8, v0 - MOVW v0, (0xbfc007fc) // magic N64 hardware init - - // a bit from libgloss so we start at a known state - MOVW $DEFAULT_C0_SR, v0 - MOVW v0, M(C0_SR) - MOVW $0, M(C0_CAUSE) - - // Check if PI DMA transfer is required, knowing that IPL3 loads 1 MiB - // of ROM to RAM, and __libdragon_text_start is located right after the - // ROM header where this 1 MiB starts. - // TODO test again with TLB mappings configured - MOVW $_rt0_mips64_noos(SB), a0 - MOVW $runtime·edata(SB), a1 - MOVW $0x100000, t0 // stock IPL3 load size (1 MiB) - SUBU a0, a1, a2 // calculate data size - SUB t0, a2, a2 // calculate remaining data size - BLEZ a2, skip_dma // skip PI DMA if data is already loaded - - // Copy code and data via DMA - MOVW $0x10001000, a1 // address in rom - ADDU t0, a0, a0 // skip over loaded data - ADDU t0, a1, a1 - - // Start PI DMA transfer - MOVW $0xA4600000, t0 - MOVW a0, 0x00(t0) // PI_DRAM_ADDR - MOVW a1, 0x04(t0) // PI_CART_ADDR - ADD $-1, a2 - MOVW a2, 0x0C(t0) // PI_WR_LEN - -skip_dma: + JMP ·rt0_target(SB) + +TEXT runtime·_rt0_mips64_noos1(SB),NOSPLIT|NOFRAME,$0 // fill .bss with 0s - SUBU $16, sp - MOVW $runtime·bss(SB), a0 - OR $0x20000000, a0 // convert address to KSEG1 (uncached) - MOVW $runtime·ebss(SB), a1 - OR $0x20000000, a1 - SUB a0, a1 - MOVV a0, 8(sp) - MOVV a1, 16(sp) + SUBU $16, R29 + MOVW $runtime·bss(SB), R4 + OR $0x20000000, R4 // convert address to KSEG1 (uncached) + MOVW $runtime·ebss(SB), R5 + OR $0x20000000, R5 + SUB R4, R5 + MOVV R4, 8(R29) + MOVV R5, 16(R29) JAL runtime·memclrNoHeapPointers(SB) // clear BSS // fill .noptrbss with 0s - MOVW $runtime·noptrbss(SB), a0 - OR $0x20000000, a0 // convert address to KSEG1 (uncached) - MOVW $runtime·enoptrbss(SB), a1 - OR $0x20000000, a1 - SUB a0, a1 - MOVV a0, 8(sp) - MOVV a1, 16(sp) + MOVW $runtime·noptrbss(SB), R4 + OR $0x20000000, R4 // convert address to KSEG1 (uncached) + MOVW $runtime·enoptrbss(SB), R5 + OR $0x20000000, R5 + SUB R4, R5 + MOVV R4, 8(R29) + MOVV R5, 16(R29) JAL runtime·memclrNoHeapPointers(SB) // clear noptrBSS // clear unallocated memory - MOVW $runtime·end(SB), a0 - OR $0x20000000, a0 // convert address to KSEG1 (uncached) - MOVW $runtime·ramend(SB), a1 - OR $0x20000000, a1 - SUB a0, a1 - MOVV a0, 8(sp) - MOVV a1, 16(sp) + MOVW $runtime·end(SB), R4 + OR $0x20000000, R4 // convert address to KSEG1 (uncached) + MOVW $runtime·ramend(SB), R5 + OR $0x20000000, R5 + SUB R4, R5 + MOVV R4, 8(R29) + MOVV R5, 16(R29) JAL runtime·memclrNoHeapPointers(SB) // clear unallocated memory - ADDU $16, sp - - // Wait for DMA transfer to be finished - MOVW $0xA4600000, t0 - -wait_dma_end: - MOVW 0x10(t0), t1 // PI_STATUS - AND $3, t1 // PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY - BGTZ t1, wait_dma_end - - // Store the bbplayer flag now that BSS has been cleared - MOVB t7, runtime·bbplayer(SB) + ADDU $16, R29 // load interrupt vector - MOVW $·intvector(SB), t0 - MOVW $0xa0000000, t1 - MOVW $4, t2 - -loadintvectorloop: - MOVW (t0), t3 - MOVW t3, 0(t1) - MOVW t3, 0x80(t1) - MOVW t3, 0x100(t1) - MOVW t3, 0x180(t1) + MOVW $runtime·intvector(SB), R8 + MOVW $0xa0000000, R9 + MOVW $4, R10 +loop: + MOVW (R8), R11 + MOVW R11, 0(R9) + MOVW R11, 0x80(R9) + MOVW R11, 0x100(R9) + MOVW R11, 0x180(R9) // sync - BREAK is overloaded CACHE opcode - BREAK R16, 0(t1) - BREAK R16, 0x80(t1) - BREAK R16, 0x100(t1) - BREAK R16, 0x180(t1) - ADD $4, t0 - ADD $4, t1 - ADDU $-1, t2 - BGTZ t2,loadintvectorloop + BREAK R16, 0(R9) + BREAK R16, 0x80(R9) + BREAK R16, 0x100(R9) + BREAK R16, 0x180(R9) + ADD $4, R8 + ADD $4, R9 + ADDU $-1, R10 + BGTZ R10,loop JMP runtime·rt0_go(SB) -// Maps KSEG0, KSEG1 and ROM to the beginning of the virtual address space. -// This saves us from sign-extending pointers correctly, as we avoid pointers -// upwards of 0x80000000. -// The only problems encountered with falsely sign-extended pointers were symbol -// addresses loaded from the ELF. Otherwise running code in KSEG0 seems to be -// no problem in general. -// The correct way of doing this would probably involve defining a new 64-bit -// architecture with PtrSize = 4, but I have ran into some issues that weren't -// worth fixing at the moment. -// TODO currently only 16 MiB of cartridge is mapped -TEXT runtime·rt0_tlb(SB),NOSPLIT|NOFRAME,$0 - MOVV $0, R8 - MOVV R8, M(C0_INDEX) - MOVV $0xfff << 13, R8 - MOVV R8, M(C0_PAGEMASK) - MOVV $(0x00000000 >> 6) | 0x7, R8 - MOVV R8, M(C0_ENTRYLO0) - MOVV $(0x01000000 >> 6) | 0x7, R8 - MOVV R8, M(C0_ENTRYLO1) - MOVV $0x00000000, R8 - MOVV R8, M(C0_ENTRYHI) - NOOP // avert CP0 hazards - TLBWI - - MOVV $1, R8 - MOVV R8, M(C0_INDEX) - MOVV $0xfff << 13, R8 - MOVV R8, M(C0_PAGEMASK) - MOVV $(0x10000000 >> 6) | (2<<3) | 0x3, R8 - MOVV R8, M(C0_ENTRYLO0) - MOVV $(0x11000000 >> 6) | (2<<3) | 0x3, R8 - MOVV R8, M(C0_ENTRYLO1) - MOVV $0x10000000, R8 - MOVV R8, M(C0_ENTRYHI) - NOOP // avert CP0 hazards - TLBWI - - MOVV $2, R8 - MOVV R8, M(C0_INDEX) - MOVV $0xfff << 13, R8 - MOVV R8, M(C0_PAGEMASK) - MOVV $(0x00000000 >> 6) | (2<<3) | 0x7, R8 - MOVV R8, M(C0_ENTRYLO0) - MOVV $(0x01000000 >> 6) | (2<<3) | 0x7, R8 - MOVV R8, M(C0_ENTRYLO1) - MOVV $0x20000000, R8 - MOVV R8, M(C0_ENTRYHI) - NOOP // avert CP0 hazards - TLBWI - - MOVV $0x7fffffff, R8 - AND R8, ra // return to the tlb mapped address - RET - - #define PALLOC_MIN 20*1024 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 diff --git a/src/runtime/target_noos_n64.go b/src/runtime/target_noos_n64.go new file mode 100644 index 00000000000000..d1e1ddf00b8f3a --- /dev/null +++ b/src/runtime/target_noos_n64.go @@ -0,0 +1,11 @@ +//go:build noos && n64 + +package runtime + +import "github.com/clktmr/n64/machine" + +//go:nowritebarrierrec +//go:nosplit +func targetDefaultWrite(fd int, p []byte) int { + return machine.DefaultWrite(fd, p) +} diff --git a/src/runtime/target_noos_n64.s b/src/runtime/target_noos_n64.s new file mode 100644 index 00000000000000..05697476d5d8d3 --- /dev/null +++ b/src/runtime/target_noos_n64.s @@ -0,0 +1,7 @@ +#include "go_asm.h" +#include "go_tls.h" +#include "funcdata.h" +#include "textflag.h" + +TEXT ·rt0_target(SB),NOSPLIT|NOFRAME,$0 + JMP machine·rt0(SB) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index 05750dd407a312..16c5261381143d 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -23,8 +23,6 @@ type mOS struct { sp, fp, ra, epc uintptr } -var bbplayer bool // TODO move to n64 board support package - var ( cpu0 cpuctx pcpu0 = &cpu0 @@ -121,32 +119,18 @@ func curcpuSchedule() { curcpu().schedule = true } +//go:nowritebarrierrec +//go:nosplit +func defaultWrite(fd int, p []byte) int { + return targetDefaultWrite(fd, p) +} + // only called from ISR, do not use func enterScheduler() func saveGPRs() func restoreGPRs() func saveFPRs() func restoreFPRs() - -var lastlog [1024 * 16]byte -var lastidx = 0 - -// Just write into some ringbuffer for now, which can be inspected in a memory -// dump. -// -//go:nowritebarrierrec -//go:nosplit -func defaultWrite(fd int, p []byte) int { - for i := 0; i < len(p); i++ { - lastlog[lastidx] = p[i] - lastidx += 1 - if lastidx >= len(lastlog) { - lastidx = 0 - } - } - return len(p) -} - func unhandledExternalInterrupt() // syscalls not used by runtime From 738945eda65bdf23d6593ae97abc5dc4f8155940 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 13 Jun 2024 01:50:53 +0200 Subject: [PATCH 32/45] Remove incorrect uncached memory accesses Before clearing the memory via uncached access, all cache must be invalidated before. Otherwise a cache writeback could happen after clearing the memory. --- src/runtime/rt0_noos_mips64.s | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 93224edf5a3ee9..0678c8d2af7f38 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -8,43 +8,40 @@ #include "textflag.h" #include "asm_mips64.h" +// BREAK is overloaded CACHE opcode. Register number specifies the cache op. +#define CACHE BREAK + TEXT _rt0_mips64_noos(SB),NOSPLIT|NOFRAME,$0 JMP ·rt0_target(SB) TEXT runtime·_rt0_mips64_noos1(SB),NOSPLIT|NOFRAME,$0 - // fill .bss with 0s + // Clear .bss, .noptrbss and unallocated memory. SUBU $16, R29 + MOVW $runtime·bss(SB), R4 - OR $0x20000000, R4 // convert address to KSEG1 (uncached) MOVW $runtime·ebss(SB), R5 - OR $0x20000000, R5 SUB R4, R5 MOVV R4, 8(R29) MOVV R5, 16(R29) - JAL runtime·memclrNoHeapPointers(SB) // clear BSS + JAL runtime·memclrNoHeapPointers(SB) - // fill .noptrbss with 0s MOVW $runtime·noptrbss(SB), R4 - OR $0x20000000, R4 // convert address to KSEG1 (uncached) MOVW $runtime·enoptrbss(SB), R5 - OR $0x20000000, R5 SUB R4, R5 MOVV R4, 8(R29) MOVV R5, 16(R29) - JAL runtime·memclrNoHeapPointers(SB) // clear noptrBSS + JAL runtime·memclrNoHeapPointers(SB) - // clear unallocated memory MOVW $runtime·end(SB), R4 - OR $0x20000000, R4 // convert address to KSEG1 (uncached) MOVW $runtime·ramend(SB), R5 - OR $0x20000000, R5 SUB R4, R5 MOVV R4, 8(R29) MOVV R5, 16(R29) - JAL runtime·memclrNoHeapPointers(SB) // clear unallocated memory + JAL runtime·memclrNoHeapPointers(SB) + ADDU $16, R29 - // load interrupt vector + // Load interrupt vector MOVW $runtime·intvector(SB), R8 MOVW $0xa0000000, R9 MOVW $4, R10 @@ -54,11 +51,10 @@ loop: MOVW R11, 0x80(R9) MOVW R11, 0x100(R9) MOVW R11, 0x180(R9) - // sync - BREAK is overloaded CACHE opcode - BREAK R16, 0(R9) - BREAK R16, 0x80(R9) - BREAK R16, 0x100(R9) - BREAK R16, 0x180(R9) + CACHE R16, 0(R9) + CACHE R16, 0x80(R9) + CACHE R16, 0x100(R9) + CACHE R16, 0x180(R9) ADD $4, R8 ADD $4, R9 ADDU $-1, R10 From 225f0caa882a8ce9ede718b7b5e5a214b28db2fc Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 13 Jun 2024 01:59:07 +0200 Subject: [PATCH 33/45] Use Go register notation --- src/runtime/rt0_noos_mips64.s | 136 +++++++++++++++++----------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 0678c8d2af7f38..4f73124bcd85b1 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -67,118 +67,118 @@ loop: TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 // setup main stack in cpu0.gh - MOVV $runtime·cpu0(SB), t0 // gh is the first field of the cpuctx struct - MOVV $runtime·ramend(SB), sp // main stack starts at the end of memory - SUB $16, sp - MOVV sp, (g_stack+stack_hi)(t0) - SUB $4096, sp, t1 - MOVV t1, (g_stack+stack_lo)(t0) - ADD $const_stackGuard, t1 - MOVV t1, g_stackguard0(t0) - MOVV t1, g_stackguard1(t0) + MOVV $runtime·cpu0(SB), R8 // gh is the first field of the cpuctx struct + MOVV $runtime·ramend(SB), R29 // main stack starts at the end of memory + SUB $16, R29 + MOVV R29, (g_stack+stack_hi)(R8) + SUB $4096, R29, R9 + MOVV R9, (g_stack+stack_lo)(R8) + ADD $const_stackGuard, R9 + MOVV R9, g_stackguard0(R8) + MOVV R9, g_stackguard1(R8) // set up m0 (bootstrap thread), temporarily use cpu0.gh as g - MOVV $runtime·m0(SB), t1 - MOVV t0, m_g0(t1) // m0.g0 = cpu0.gh - MOVV t1, g_m(t0) // cpu0.gh.m = m0 + MOVV $runtime·m0(SB), R9 + MOVV R8, m_g0(R9) // m0.g0 = cpu0.gh + MOVV R9, g_m(R8) // cpu0.gh.m = m0 - MOVV t0, g // set g to gh + MOVV R8, g // set g to gh JAL runtime·check(SB) JAL runtime·osinit(SB) // initialize noosMem - MOVV $runtime·end(SB), t0 - MOVV $runtime·ramend(SB), t1 - SUB $4096, t1 - SUB t0, t1, t5 // size of available memory (DMA capable) + MOVV $runtime·end(SB), R8 + MOVV $runtime·ramend(SB), R9 + SUB $4096, R9 + SUB R8, R9, R13 // size of available memory (DMA capable) // estimate the space need for non-heap allocations - SRL $(const__PageShift), t5, t4 - MOVV $mspan__size, t2 - MUL t2, t4 - MOVV LO, t4 - ADD $PALLOC_MIN, t4 + SRL $(const__PageShift), R13, R12 + MOVV $mspan__size, R10 + MUL R10, R12 + MOVV LO, R12 + ADD $PALLOC_MIN, R12 - MOVV $runtime·nodmastart(SB), t2 - MOVV $runtime·nodmaend(SB), t3 - SUB t2, t3, t7 // size of non-DMA memory - ADD t5, t7, t6 // size of the whole free memory + MOVV $runtime·nodmastart(SB), R10 + MOVV $runtime·nodmaend(SB), R11 + SUB R10, R11, R15 // size of non-DMA memory + ADD R13, R15, R14 // size of the whole free memory // we prefer the non-DMA memory for non-heap objects to preserve as much as // possible of the DMA capable memory for heap allocations - SUB t7, t4 + SUB R15, R12 // reduce the arena by the remain of the non-heap space that did not fit in // the non-DMA memory, properly align the arena - BLTZ t4, 2(PC) - SUB t4, t5 - AND $~(const_heapArenaBytes-1), t5 - SUB t5, t1 - MOVV t1, t4 + BLTZ R12, 2(PC) + SUB R12, R13 + AND $~(const_heapArenaBytes-1), R13 + SUB R13, R9 + MOVV R9, R12 // save {free.start,free.end,nodma.start,nodma.end,arenaStart,arenaSize,size} - MOVV $runtime·noosMem(SB), t7 - MOVV t0, 0(t7) - MOVV t1, 8(t7) - MOVV t2, 16(t7) - MOVV t3, 24(t7) - MOVV t4, 32(t7) - MOVV t5, 40(t7) - MOVV t6, 48(t7) + MOVV $runtime·noosMem(SB), R15 + MOVV R8, 0(R15) + MOVV R9, 8(R15) + MOVV R10, 16(R15) + MOVV R11, 24(R15) + MOVV R12, 32(R15) + MOVV R13, 40(R15) + MOVV R14, 48(R15) // initialize noos tasker and Go scheduler JAL runtime·taskerinit(SB) JAL runtime·schedinit(SB) // allocate g0 for m0 and leave gh - SUB $16, sp - MOVW $(2*const_stackMin), a0 - MOVW a0, 8(sp) + SUB $16, R29 + MOVW $(2*const_stackMin), R4 + MOVW R4, 8(R29) JAL runtime·malg(SB) - MOVV 16(sp), t0 // newg in t0 - ADD $16, sp + MOVV 16(R29), R8 // newg in R8 + ADD $16, R29 // stackguard check during newproc requires valid stackguard1 but // malg sets it to 0xFFFFFFFF (mstart fixes this but is called later) - MOVV g_stackguard0(t0), t1 - MOVV t1, g_stackguard1(t0) + MOVV g_stackguard0(R8), R9 + MOVV R9, g_stackguard1(R8) - MOVV $runtime·m0(SB), t1 - MOVV t0, m_g0(t1) // m0.g0 = newg - MOVV t1, g_m(t0) // newg.m = m0 + MOVV $runtime·m0(SB), R9 + MOVV R8, m_g0(R9) // m0.g0 = newg + MOVV R9, g_m(R8) // newg.m = m0 - // newg stack pointer to sp - MOVV (g_stack+stack_hi)(t0), sp + // newg stack pointer to R29 + MOVV (g_stack+stack_hi)(R8), R29 // newg to g - MOVV g, t2 - MOVV t0, g + MOVV g, R10 + MOVV R8, g // fix cpu0.gh, cpu0.mh - ADD $cpuctx_mh, t2, t1 // t2 points to cpu0 (and to cpu0.gh at the same time) - MOVV t2, m_g0(t1) // cpu0.mh.g0 = cpu0.gh - MOVV t2, m_gsignal(t1) // cpu0.mh.gsignal = cpu0.gh (to easily check for handler mode) - MOVV t1, g_m(t2) // cpu0.gh.m = cpu0.mh + ADD $cpuctx_mh, R10, R9 // R10 points to cpu0 (and to cpu0.gh at the same time) + MOVV R10, m_g0(R9) // cpu0.mh.g0 = cpu0.gh + MOVV R10, m_gsignal(R9) // cpu0.mh.gsignal = cpu0.gh (to easily check for handler mode) + MOVV R9, g_m(R10) // cpu0.gh.m = cpu0.mh // TODO switch to the user mode? // create a new goroutine to start program - SUB $16, sp - MOVV $runtime·mainPC(SB), t0 - MOVV t0, 8(sp) // arg 1: fn - MOVV $0, t0 - MOVV t0, 0(sp) // dummy LR + SUB $16, R29 + MOVV $runtime·mainPC(SB), R8 + MOVV R8, 8(R29) // arg 1: fn + MOVV $0, R8 + MOVV R8, 0(R29) // dummy LR JAL runtime·newproc(SB) - ADD $16, sp // pop args and LR + ADD $16, R29 // pop args and LR // enable interrupts // TODO where to enable interupts correctly? - MOVW z0, M(C0_COMPARE) - MOVW M(C0_SR), t0 - OR $(SR_IE|INTR_SW|INTR_EXT), t0 - MOVW t0, M(C0_SR) + MOVW R0, M(C0_COMPARE) + MOVW M(C0_SR), R8 + OR $(SR_IE|INTR_SW|INTR_EXT), R8 + MOVW R8, M(C0_SR) // start this M JAL runtime·mstart(SB) From d4607cafec2fa6da59148426ed70d8f9380a6ef1 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 13 Jun 2024 14:50:10 +0200 Subject: [PATCH 34/45] Implement meminit in Go instead of assembly --- src/runtime/rt0_noos_mips64.go | 33 ++++++++++++++++++ src/runtime/rt0_noos_mips64.s | 62 ++++++++++------------------------ 2 files changed, 51 insertions(+), 44 deletions(-) create mode 100644 src/runtime/rt0_noos_mips64.go diff --git a/src/runtime/rt0_noos_mips64.go b/src/runtime/rt0_noos_mips64.go new file mode 100644 index 00000000000000..850a42528e8867 --- /dev/null +++ b/src/runtime/rt0_noos_mips64.go @@ -0,0 +1,33 @@ +package runtime + +import "unsafe" + +const pallocMin = 20 * 1024 + +//go:nosplit +func meminit(freeStart, freeEnd, nodmaStart, nodmaEnd uintptr) { + freeSize := freeEnd - freeStart + nodmaSize := nodmaEnd - nodmaStart + size := freeSize + nodmaSize + + // Estimate the space needed for non-heap allocations + palloc := int((freeSize>>(pageShift))*unsafe.Sizeof(emptymspan) + pallocMin) + + // We prefer the non-DMA memory for non-heap objects to preserve as much + // as possible of the DMA capable memory for heap allocations + pallocInFree := max(palloc-int(nodmaSize), 0) + + // Reduce the arena by the remainder of the non-heap space that did not fit in + // the non-DMA memory, properly align the arena + arenaSize := freeSize - uintptr(pallocInFree) + arenaSize &= ^(uintptr(heapArenaBytes) - 1) + arenaStart := freeEnd - arenaSize + + noosMem.free.start = freeStart + noosMem.free.end = freeEnd + noosMem.nodma.start = nodmaStart + noosMem.nodma.end = nodmaEnd + noosMem.arenaStart = arenaStart + noosMem.arenaSize = arenaSize + noosMem.size = size +} diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index 4f73124bcd85b1..b2d575de9f5f2b 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -84,49 +84,23 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVV R8, g // set g to gh - JAL runtime·check(SB) - JAL runtime·osinit(SB) + JAL runtime·check(SB) + JAL runtime·osinit(SB) // initialize noosMem - MOVV $runtime·end(SB), R8 MOVV $runtime·ramend(SB), R9 - SUB $4096, R9 - SUB R8, R9, R13 // size of available memory (DMA capable) - - // estimate the space need for non-heap allocations - SRL $(const__PageShift), R13, R12 - MOVV $mspan__size, R10 - MUL R10, R12 - MOVV LO, R12 - ADD $PALLOC_MIN, R12 - MOVV $runtime·nodmastart(SB), R10 MOVV $runtime·nodmaend(SB), R11 - SUB R10, R11, R15 // size of non-DMA memory - ADD R13, R15, R14 // size of the whole free memory - - // we prefer the non-DMA memory for non-heap objects to preserve as much as - // possible of the DMA capable memory for heap allocations - SUB R15, R12 - - // reduce the arena by the remain of the non-heap space that did not fit in - // the non-DMA memory, properly align the arena - BLTZ R12, 2(PC) - SUB R12, R13 - AND $~(const_heapArenaBytes-1), R13 - SUB R13, R9 - MOVV R9, R12 - - // save {free.start,free.end,nodma.start,nodma.end,arenaStart,arenaSize,size} - MOVV $runtime·noosMem(SB), R15 - MOVV R8, 0(R15) - MOVV R9, 8(R15) - MOVV R10, 16(R15) - MOVV R11, 24(R15) - MOVV R12, 32(R15) - MOVV R13, 40(R15) - MOVV R14, 48(R15) + SUB $4096, R9 + + SUB $32, R29 + MOVV R8, 8(R29) + MOVV R9, 16(R29) + MOVV R10, 24(R29) + MOVV R11, 32(R29) + JAL runtime·meminit(SB) + ADD $32, R29 // initialize noos tasker and Go scheduler JAL runtime·taskerinit(SB) @@ -165,13 +139,13 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 // TODO switch to the user mode? // create a new goroutine to start program - SUB $16, R29 - MOVV $runtime·mainPC(SB), R8 - MOVV R8, 8(R29) // arg 1: fn - MOVV $0, R8 - MOVV R8, 0(R29) // dummy LR - JAL runtime·newproc(SB) - ADD $16, R29 // pop args and LR + SUB $16, R29 + MOVV $runtime·mainPC(SB), R8 + MOVV R8, 8(R29) // arg 1: fn + MOVV $0, R8 + MOVV R8, 0(R29) // dummy LR + JAL runtime·newproc(SB) + ADD $16, R29 // pop args and LR // enable interrupts // TODO where to enable interupts correctly? From 4d92f7050d49b8a03c87dbf7985a06e5d13ecb80 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 13 Jun 2024 17:35:27 +0200 Subject: [PATCH 35/45] Cleanup asm_mips64.h --- src/runtime/asm_mips64.h | 275 +++++++++++-------------------- src/runtime/rt0_noos_mips64.go | 1 + src/runtime/rt0_noos_mips64.s | 27 ++- src/runtime/tasker_noos_mips64.s | 7 - 4 files changed, 111 insertions(+), 199 deletions(-) diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index c2a2825841492e..b178402f4ec153 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -1,177 +1,102 @@ -/* - * standard MIPS register names. - * - * Copyright (c) 1995 Cygnus Support - * - * The authors hereby grant permission to use, copy, modify, distribute, - * and license this software and its documentation for any purpose, provided - * that existing copyright notices are retained in all copies and that this - * notice is included verbatim in any distributions. No written agreement, - * license, or royalty fee is required for any of the authorized uses. - * Modifications to this software may be copyrighted by their authors - * and need not follow the licensing terms described here, provided that - * the new terms are clearly indicated on the first page of each file where - * they apply. - */ - -/* Standard MIPS register names: */ -#define zero R0 -#define z0 R0 -#define at R1 -#define v0 R2 -#define v1 R3 -#define a0 R4 -#define a1 R5 -#define a2 R6 -#define a3 R7 -#define t0 R8 -#define t1 R9 -#define t2 R10 -#define t3 R11 -#define t4 R12 -#define t5 R13 -#define t6 R14 -#define t7 R15 -#define s0 R16 -#define s1 R17 -#define s2 R18 -#define s3 R19 -#define s4 R20 -#define s5 R21 -#define s6 R22 -#define s7 R23 -#define t8 R24 -#define t9 R25 -#define k0 R26 /* kernel private register 0 */ -#define k1 R27 /* kernel private register 1 */ -#define gp R28 /* global data pointer */ -#define sp R29 /* stack-pointer */ -#define fp R30 /* frame-pointer */ -#define ra R31 /* return address */ - -#define fp0 F0 -#define fp1 F1 - -/* Useful memory constants: */ -#define K0BASE 0x80000000 -#ifndef __mips64 -#define K1BASE 0xA0000000 -#define K0BASE_ADDR ((char *)K0BASE) -#define K1BASE_ADDR ((char *)K1BASE) -#else -#define K1BASE 0xFFFFFFFFA0000000LL -#define K0BASE_ADDR ((char *)0xFFFFFFFF80000000LL) -#define K1BASE_ADDR ((char *)K1BASE) -#endif - -#define PHYS_TO_K1(a) ((unsigned)(a) | K1BASE) - -/* Standard Co-Processor 0 registers */ -#define C0_INDEX 0 /* Index of TLB Entry */ -#define C0_ENTRYLO0 2 /* TLB entry's first PFN */ -#define C0_ENTRYLO1 3 /* TLB entry's second PFN */ -#define C0_PAGEMASK 5 /* Size of TLB Entries */ -#define C0_COUNT 9 /* Timer Count Register */ -#define C0_ENTRYHI 10 /* VPN and ASID of two TLB entry */ -#define C0_COMPARE 11 /* Timer Compare Register */ -#define C0_SR 12 /* Status Register */ -#define C0_CAUSE 13 /* last exception description */ -#define C0_EPC 14 /* Exception error address */ -#define C0_PRID 15 /* Processor Revision ID */ -#define C0_CONFIG 16 /* CPU configuration */ -#define C0_WATCHLO 18 /* Watchpoint */ - -/* Standard Processor Revision ID Register field offsets */ -#define PR_IMP 8 - -/* Standard Config Register field offsets */ -#define CR_DB 4 -#define CR_IB 5 -#define CR_DC 6 /* NOTE v4121 semantics != 43,5xxx semantics */ -#define CR_IC 9 /* NOTE v4121 semantics != 43,5xxx semantics */ -#define CR_SC 17 -#define CR_SS 20 -#define CR_SB 22 - - -/* Standard Status Register bitmasks: */ -#define SR_CU1 0x20000000 /* Mark CP1 as usable */ -#define SR_FR 0x04000000 /* Enable MIPS III FP registers */ -#define SR_BEV 0x00400000 /* Controls location of exception vectors */ -#define SR_PE 0x00100000 /* Mark soft reset (clear parity error) */ - -#define SR_KX 0x00000080 /* Kernel extended addressing enabled */ -#define SR_SX 0x00000040 /* Supervisor extended addressing enabled */ -#define SR_UX 0x00000020 /* User extended addressing enabled */ - -#define SR_ERL 0x00000004 /* Error level */ -#define SR_EXL 0x00000002 /* Exception level */ -#define SR_IE 0x00000001 /* Interrupts enabled */ - -/* Standard Cause Register bitmasks: */ -#define CAUSE_EXC_MASK (0x1F << 2) -#define CAUSE_EXC_INTERRUPT (0 << 2) -#define CAUSE_EXC_SYSCALL (8 << 2) -#define CAUSE_EXC_BREAKPOINT (9 << 2) -#define CAUSE_EXC_COPROCESSOR (11 << 2) +// Copyright (c) 1995 Cygnus Support +// +// The authors hereby grant permission to use, copy, modify, distribute, and +// license this software and its documentation for any purpose, provided that +// existing copyright notices are retained in all copies and that this notice is +// included verbatim in any distributions. No written agreement, license, or +// royalty fee is required for any of the authorized uses. Modifications to this +// software may be copyrighted by their authors and need not follow the +// licensing terms described here, provided that the new terms are clearly +// indicated on the first page of each file where they apply. + + +// Useful memory constants +#define KSEG0 0xFFFFFFFF80000000 +#define KSEG1 0xFFFFFFFFA0000000 + +// Standard Co-Processor 0 registers +#define C0_INDEX 0 // Index of TLB Entry +#define C0_ENTRYLO0 2 // TLB entry's first PFN +#define C0_ENTRYLO1 3 // TLB entry's second PFN +#define C0_PAGEMASK 5 // Size of TLB Entries +#define C0_COUNT 9 // Timer Count Register +#define C0_ENTRYHI 10 // VPN and ASID of two TLB entry +#define C0_COMPARE 11 // Timer Compare Register +#define C0_SR 12 // Status Register +#define C0_CAUSE 13 // last exception description +#define C0_EPC 14 // Exception error address +#define C0_PRID 15 // Processor Revision ID +#define C0_CONFIG 16 // CPU configuration +#define C0_WATCHLO 18 // Watchpoint + +// Standard Processor Revision ID Register field offsets +#define PR_IMP 8 + +// Standard Config Register field offsets +#define CR_DB 4 +#define CR_IB 5 +#define CR_DC 6 +#define CR_IC 9 +#define CR_SC 17 +#define CR_SS 20 +#define CR_SB 22 + +// Standard Status Register bitmasks +#define SR_CU1 0x20000000 // Mark CP1 as usable +#define SR_FR 0x04000000 // Enable MIPS III FP registers +#define SR_BEV 0x00400000 // Controls location of exception vectors +#define SR_PE 0x00100000 // Mark soft reset (clear parity error) + +#define SR_KX 0x00000080 // Kernel extended addressing enabled +#define SR_SX 0x00000040 // Supervisor extended addressing enabled +#define SR_UX 0x00000020 // User extended addressing enabled + +#define SR_ERL 0x00000004 // Error level +#define SR_EXL 0x00000002 // Exception level +#define SR_IE 0x00000001 // Interrupts enabled + +// Standard Cause Register bitmasks +#define CAUSE_EXC_MASK (0x1F << 2) +#define CAUSE_EXC_INTERRUPT (0 << 2) +#define CAUSE_EXC_SYSCALL (8 << 2) +#define CAUSE_EXC_BREAKPOINT (9 << 2) +#define CAUSE_EXC_COPROCESSOR (11 << 2) // Masks for Status Registers IM bits and Cause Registers IP bits -#define INTR_MASK (0xFF << 8) -#define INTR_SW (0x03 << 8) -#define INTR_EXT (0x7C << 8) -#define INTR_TIMER (0x80 << 8) - -/* Standard (R4000) cache operations. Taken from "MIPS R4000 - Microprocessor User's Manual" 2nd edition: */ - -#define CACHE_I (0) /* primary instruction */ -#define CACHE_D (1) /* primary data */ -#define CACHE_SI (2) /* secondary instruction */ -#define CACHE_SD (3) /* secondary data (or combined instruction/data) */ - -#define INDEX_INVALIDATE (0) /* also encodes WRITEBACK if CACHE_D or CACHE_SD */ -#define INDEX_LOAD_TAG (1) -#define INDEX_STORE_TAG (2) -#define CREATE_DIRTY_EXCLUSIVE (3) /* CACHE_D and CACHE_SD only */ -#define HIT_INVALIDATE (4) -#define CACHE_FILL (5) /* CACHE_I only */ -#define HIT_WRITEBACK_INVALIDATE (5) /* CACHE_D and CACHE_SD only */ -#define HIT_WRITEBACK (6) /* CACHE_I, CACHE_D and CACHE_SD only */ -#define HIT_SET_VIRTUAL (7) /* CACHE_SI and CACHE_SD only */ - -#define BUILD_CACHE_OP(o,c) (((o) << 2) | (c)) - -/* Individual cache operations: */ -#define INDEX_INVALIDATE_I BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_I) -#define INDEX_WRITEBACK_INVALIDATE_D BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_D) -#define INDEX_INVALIDATE_SI BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_SI) -#define INDEX_WRITEBACK_INVALIDATE_SD BUILD_CACHE_OP(INDEX_INVALIDATE,CACHE_SD) - -#define INDEX_LOAD_TAG_I BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_I) -#define INDEX_LOAD_TAG_D BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_D) -#define INDEX_LOAD_TAG_SI BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_SI) -#define INDEX_LOAD_TAG_SD BUILD_CACHE_OP(INDEX_LOAD_TAG,CACHE_SD) - -#define INDEX_STORE_TAG_I BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_I) -#define INDEX_STORE_TAG_D BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_D) -#define INDEX_STORE_TAG_SI BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_SI) -#define INDEX_STORE_TAG_SD BUILD_CACHE_OP(INDEX_STORE_TAG,CACHE_SD) - -#define CREATE_DIRTY_EXCLUSIVE_D BUILD_CACHE_OP(CREATE_DIRTY_EXCLUSIVE,CACHE_D) -#define CREATE_DIRTY_EXCLUSIVE_SD BUILD_CACHE_OP(CREATE_DIRTY_EXCLUSIVE,CACHE_SD) - -#define HIT_INVALIDATE_I BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_I) -#define HIT_INVALIDATE_D BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_D) -#define HIT_INVALIDATE_SI BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_SI) -#define HIT_INVALIDATE_SD BUILD_CACHE_OP(HIT_INVALIDATE,CACHE_SD) - -#define CACHE_FILL_I BUILD_CACHE_OP(CACHE_FILL,CACHE_I) -#define HIT_WRITEBACK_INVALIDATE_D BUILD_CACHE_OP(HIT_WRITEBACK_INVALIDATE,CACHE_D) -#define HIT_WRITEBACK_INVALIDATE_SD BUILD_CACHE_OP(HIT_WRITEBACK_INVALIDATE,CACHE_SD) - -#define HIT_WRITEBACK_I BUILD_CACHE_OP(HIT_WRITEBACK,CACHE_I) -#define HIT_WRITEBACK_D BUILD_CACHE_OP(HIT_WRITEBACK,CACHE_D) -#define HIT_WRITEBACK_SD BUILD_CACHE_OP(HIT_WRITEBACK,CACHE_SD) - -#define HIT_SET_VIRTUAL_SI BUILD_CACHE_OP(HIT_SET_VIRTUAL,CACHE_SI) -#define HIT_SET_VIRTUAL_SD BUILD_CACHE_OP(HIT_SET_VIRTUAL,CACHE_SD) +#define INTR_MASK (0xFF << 8) +#define INTR_SW (0x03 << 8) +#define INTR_EXT (0x7C << 8) +#define INTR_TIMER (0x80 << 8) + +// BREAK is overloaded CACHE opcode. Register number specifies the cache op. +#define CACHE BREAK +// Prepend NOOP to ERET to avert CP0 hazards +#define ERET NOOP; NOOP; WORD $0x42000018 + +// Individual cache operations +#define INDEX_INVALIDATE_I R00 +#define INDEX_WRITEBACK_INVALIDATE_D R01 +#define INDEX_INVALIDATE_SI R02 +#define INDEX_WRITEBACK_INVALIDATE_SD R03 +#define INDEX_LOAD_TAG_I R04 +#define INDEX_LOAD_TAG_D R05 +#define INDEX_LOAD_TAG_SI R06 +#define INDEX_LOAD_TAG_SD R07 +#define INDEX_STORE_TAG_I R08 +#define INDEX_STORE_TAG_D R09 +#define INDEX_STORE_TAG_SI R10 +#define INDEX_STORE_TAG_SD R11 +#define CREATE_DIRTY_EXCLUSIVE_D R13 +#define CREATE_DIRTY_EXCLUSIVE_SD R15 +#define HIT_INVALIDATE_I R16 +#define HIT_INVALIDATE_D R17 +#define HIT_INVALIDATE_SI R18 +#define HIT_INVALIDATE_SD R19 +#define CACHE_FILL_I R20 +#define HIT_WRITEBACK_INVALIDATE_D R21 +#define HIT_WRITEBACK_INVALIDATE_SD R23 +#define HIT_WRITEBACK_I R24 +#define HIT_WRITEBACK_D R25 +#define HIT_WRITEBACK_SD R27 +#define HIT_SET_VIRTUAL_SI R30 +#define HIT_SET_VIRTUAL_SD R31 diff --git a/src/runtime/rt0_noos_mips64.go b/src/runtime/rt0_noos_mips64.go index 850a42528e8867..7e215abf957a31 100644 --- a/src/runtime/rt0_noos_mips64.go +++ b/src/runtime/rt0_noos_mips64.go @@ -3,6 +3,7 @@ package runtime import "unsafe" const pallocMin = 20 * 1024 +const signalStackSize = 4096 //go:nosplit func meminit(freeStart, freeEnd, nodmaStart, nodmaEnd uintptr) { diff --git a/src/runtime/rt0_noos_mips64.s b/src/runtime/rt0_noos_mips64.s index b2d575de9f5f2b..c9a375a30a975c 100644 --- a/src/runtime/rt0_noos_mips64.s +++ b/src/runtime/rt0_noos_mips64.s @@ -8,9 +8,6 @@ #include "textflag.h" #include "asm_mips64.h" -// BREAK is overloaded CACHE opcode. Register number specifies the cache op. -#define CACHE BREAK - TEXT _rt0_mips64_noos(SB),NOSPLIT|NOFRAME,$0 JMP ·rt0_target(SB) @@ -51,10 +48,10 @@ loop: MOVW R11, 0x80(R9) MOVW R11, 0x100(R9) MOVW R11, 0x180(R9) - CACHE R16, 0(R9) - CACHE R16, 0x80(R9) - CACHE R16, 0x100(R9) - CACHE R16, 0x180(R9) + CACHE HIT_INVALIDATE_I, 0(R9) + CACHE HIT_INVALIDATE_I, 0x80(R9) + CACHE HIT_INVALIDATE_I, 0x100(R9) + CACHE HIT_INVALIDATE_I, 0x180(R9) ADD $4, R8 ADD $4, R9 ADDU $-1, R10 @@ -63,15 +60,13 @@ loop: JMP runtime·rt0_go(SB) -#define PALLOC_MIN 20*1024 - TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 // setup main stack in cpu0.gh MOVV $runtime·cpu0(SB), R8 // gh is the first field of the cpuctx struct MOVV $runtime·ramend(SB), R29 // main stack starts at the end of memory SUB $16, R29 MOVV R29, (g_stack+stack_hi)(R8) - SUB $4096, R29, R9 + SUB $const_signalStackSize, R29, R9 MOVV R9, (g_stack+stack_lo)(R8) ADD $const_stackGuard, R9 MOVV R9, g_stackguard0(R8) @@ -92,7 +87,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVV $runtime·ramend(SB), R9 MOVV $runtime·nodmastart(SB), R10 MOVV $runtime·nodmaend(SB), R11 - SUB $4096, R9 + SUB $const_signalStackSize, R9 SUB $32, R29 MOVV R8, 8(R29) @@ -103,8 +98,8 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 ADD $32, R29 // initialize noos tasker and Go scheduler - JAL runtime·taskerinit(SB) - JAL runtime·schedinit(SB) + JAL runtime·taskerinit(SB) + JAL runtime·schedinit(SB) // allocate g0 for m0 and leave gh SUB $16, R29 @@ -136,8 +131,6 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVV R10, m_gsignal(R9) // cpu0.mh.gsignal = cpu0.gh (to easily check for handler mode) MOVV R9, g_m(R10) // cpu0.gh.m = cpu0.mh - // TODO switch to the user mode? - // create a new goroutine to start program SUB $16, R29 MOVV $runtime·mainPC(SB), R8 @@ -145,13 +138,13 @@ TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVV $0, R8 MOVV R8, 0(R29) // dummy LR JAL runtime·newproc(SB) - ADD $16, R29 // pop args and LR + ADD $16, R29 // enable interrupts // TODO where to enable interupts correctly? MOVW R0, M(C0_COMPARE) MOVW M(C0_SR), R8 - OR $(SR_IE|INTR_SW|INTR_EXT), R8 + OR $(SR_IE|INTR_SW|INTR_EXT), R8 MOVW R8, M(C0_SR) // start this M diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 8fe8431130900f..318b25ec8ab663 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -10,7 +10,6 @@ #include "asm_mips64.h" #define sysMaxArgs (48+8) -#define ERET WORD $0x42000018 // Exception Context #define _LR (0*8) @@ -236,8 +235,6 @@ nothingToCopy: MOVV (g_sched+gobuf_g)(R26), g return: - NOOP // avert CP0 hazards - NOOP ERET @@ -322,8 +319,6 @@ smallCtx: MOVV (m_mOS+mOS_epc)(R27), R26 MOVV R26, M(C0_EPC) - NOOP // avert CP0 hazards - NOOP ERET @@ -418,8 +413,6 @@ callVector: MOVV (g_sched+gobuf_g)(R26), g return: - NOOP // avert CP0 hazards - NOOP ERET fatal: From 53f623215fa9bd658f13a1473909d11232da327c Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Tue, 18 Jun 2024 13:43:21 +0200 Subject: [PATCH 36/45] Allow target specific unhandled exception handler --- src/runtime/sys_noos_mips64.s | 2 +- src/runtime/target_noos_n64.s | 3 +++ src/runtime/tasker_noos_mips64.s | 4 +--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/runtime/sys_noos_mips64.s b/src/runtime/sys_noos_mips64.s index 0e30655aea3ec7..2ed4db573c7c30 100644 --- a/src/runtime/sys_noos_mips64.s +++ b/src/runtime/sys_noos_mips64.s @@ -147,5 +147,5 @@ TEXT ·reset(SB),NOSPLIT|NOFRAME,$0-16 // func exit(r int32) TEXT ·exit(SB),NOSPLIT|NOFRAME,$0-8 - BREAK + NOOP JMP -1(PC) diff --git a/src/runtime/target_noos_n64.s b/src/runtime/target_noos_n64.s index 05697476d5d8d3..0a2adae93e22ff 100644 --- a/src/runtime/target_noos_n64.s +++ b/src/runtime/target_noos_n64.s @@ -5,3 +5,6 @@ TEXT ·rt0_target(SB),NOSPLIT|NOFRAME,$0 JMP machine·rt0(SB) + +TEXT ·unhandledExcepton_target(SB),NOSPLIT|NOFRAME,$0 + JMP machine·unhandledException(SB) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 318b25ec8ab663..a71e1dbf5cf8a1 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -117,9 +117,7 @@ fromHandler: JMP ·softwareInterruptHandler(SB) fatal: - // unhandled exception - BREAK - JMP -1(PC) + JMP ·unhandledExcepton_target(SB) // System call is like oridnary function call so all registers are caller save From 2cb28295ed09909a4197e538d92fcab4f4d3b6a8 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 19 Jun 2024 11:45:56 +0200 Subject: [PATCH 37/45] Cleanup tasker implementation --- src/runtime/sys_noos_mips64.s | 2 - src/runtime/tasker_noos_mips64.go | 36 +------- src/runtime/tasker_noos_mips64.s | 134 +++++++++++++++--------------- 3 files changed, 69 insertions(+), 103 deletions(-) diff --git a/src/runtime/sys_noos_mips64.s b/src/runtime/sys_noos_mips64.s index 2ed4db573c7c30..912b8aade44b64 100644 --- a/src/runtime/sys_noos_mips64.s +++ b/src/runtime/sys_noos_mips64.s @@ -143,8 +143,6 @@ TEXT ·reset(SB),NOSPLIT|NOFRAME,$0-16 SYSCALL RET -// unsupported syscalls - // func exit(r int32) TEXT ·exit(SB),NOSPLIT|NOFRAME,$0-8 NOOP diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index 16c5261381143d..1d1853c3181cb4 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -10,7 +10,7 @@ import ( "unsafe" ) -// see saveGPRS and saveFPRS +// See saveGPRS and saveFPRS const ( numGPRS = 28 numFPRS = 33 @@ -39,9 +39,6 @@ func taskerinit() { allcpu.cap = 1 } -// This functions is called to put the CPU to sleep. It is allowed it does -// nothing. -// //go:nosplit func curcpuSleep() { // Check defensively for external interrupts here. There were lockups @@ -52,12 +49,6 @@ func curcpuSleep() { creg.STATUS.ClearBits(creg.IP_EXT) } -// This function is used to inform the another CPU that there is a new thread -// added to its runnable queue. It should wake up the sleeping CPU or preempt -// the currently running thread to run the scheduler. The thread preemption can -// be set as delayed to allow a running thread to run for a minimum period of -// time. -// //go:nosplit func (cpu *cpuctx) newwork() { // TODO use only one of the two interrupts @@ -69,17 +60,11 @@ func curcpuWakeup() { // TODO implement } -// This function is called to save the remaining context, not saved at syscall -// entry (eg. it can save FPU state). -// //go:nosplit func curcpuSavectxSched() { // TODO implement } -// This function is called to save the remaining context, not saved at syscall -// entry (eg. it can save FPU state). -// //go:nosplit func curcpuSavectxCall() { // TODO implement @@ -91,9 +76,6 @@ func cpuid() int { return 0 } -// This function is called to create the inintial state of the new thread and -// save it in provided m. -// //go:nosplit func archnewm(m *m) { m.epc = abi.FuncPCABI0(mstart) @@ -102,16 +84,6 @@ func archnewm(m *m) { m.ra = 1 // smallCtx flag } -// Run scheduler immediately or at syscall exit. It's called only just before -// syscall exit. -// -// The actual context switch is performed by architecture specific code at -// curcpuRunScheduler exit. It should check the cpuctx.newexe variable and if -// true switch the context to the new thread specified in cpuctx.exe. -// -// Tasker code does not use FPU so the architecture specific context switch -// code can avoid saving/restoring FPU context if not need. -// //go:nosplit func curcpuSchedule() { // syscall still needs to duffcopy it's result back. mark for schedule @@ -133,6 +105,6 @@ func saveFPRs() func restoreFPRs() func unhandledExternalInterrupt() -// syscalls not used by runtime -func syssetprivlevel(newlevel int) (oldlevel, errno int) { return } // TODO -func sysirqctl(irq, ctl, ctxid int) (enabled, prio, errno int) { return } // TODO +// syscalls unsupported by mips64 +func syssetprivlevel(newlevel int) (oldlevel, errno int) { return } +func sysirqctl(irq, ctl, ctxid int) (enabled, prio, errno int) { return } diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index a71e1dbf5cf8a1..47b58df0e30b9e 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -21,29 +21,29 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 JMP ·exceptionTrap(SB) -// main exception handler +// Main exception handler. // // R26 and R27 are free, see runtime/asm_mips64x.s // -// interrupt exceptions are disabled by EXL=1 but other exceptions can +// Interrupt exceptions are disabled by EXL=1 but other exceptions can // still occur. // // Only syscalls and interrupts are handled at the moment, all other exceptions // are fatal. // // Called from thread: -// 1. save goroutine context in g.sched (sp, fp) -// 2. switch to ISR stack -// 3. save exception context on ISR stack (ra, epc, cause) -// 4. save thread context in m.mOS (sp, fp, ra, epc) -// 5. if interrupt goto 8, if syscall continue -// 6. do syscall -// - copy args from caller thread stack to ISR stack -// - call service routine in ISR context -// - copy result from ISR stack to caller thread stack -// 7. if the syscall called curcpuSchedule, make a call to curcpuRunScheduler -// 8. if newexe, restore context from that thread (sp, fp, ra, epc) -// 9. return execution at epc +// 1. Save goroutine context in g.sched (sp, fp) +// 2. Switch to ISR stack +// 3. Save exception context on ISR stack (ra, epc, cause) +// 4. Save thread context in m.mOS (sp, fp, ra, epc) +// 5. If interrupt goto 8, if syscall continue +// 6. Do syscall +// - Copy args from caller thread stack to ISR stack +// - Call service routine in ISR context +// - Copy result from ISR stack to caller thread stack +// 7. If the syscall called curcpuSchedule, make a call to curcpuRunScheduler +// 8. If newexe, restore context from that thread (sp, fp, ra, epc) +// 9. Return execution at epc // // Called from handler: // same, but skip 1, 2, 4 @@ -60,7 +60,7 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 // - There are no callee save registers in go. Assume all gprs clobbered after // a function call. TEXT runtime·exceptionTrap(SB),NOSPLIT|NOFRAME,$0 - // determine caller stack + // Determine caller stack MOVV $·cpu0(SB), R26 BNE R26, g, fromThread MOVV $1, R27 // fromHandler flag @@ -69,16 +69,16 @@ TEXT runtime·exceptionTrap(SB),NOSPLIT|NOFRAME,$0 fromThread: MOVV $0, R27 - // save goroutine context in g.sched + // Save goroutine context in g.sched MOVV R29, (g_sched+gobuf_sp)(R26) MOVV g, (g_sched+gobuf_g)(R26) MOVV (g_stack+stack_hi)(R26), R29 - // switch to special ISR goroutine + // Switch to special ISR goroutine MOVV R26, g fromHandler: - // save exception context on ISR stack + // Save exception context on ISR stack SUB $excCtxSize, R29 OR $1, R31, R26 // encode smallCtx flag in ra MOVV R26, _LR(R29) @@ -88,13 +88,13 @@ fromHandler: OR R27, R26 // encode fromHandler flag in EPC MOVV R26, _mepc(R29) - // mask interrupts + // Mask interrupts MOVV M(C0_SR), R26 MOVV $~(INTR_SW|INTR_EXT), R27 AND R27, R26 MOVV R26, M(C0_SR) - // branch depending on exception cause + // Branch depending on exception cause MOVV M(C0_CAUSE), R26 AND $CAUSE_EXC_MASK, R26 @@ -130,15 +130,15 @@ fatal: // R10: return data size on the stack TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 MOVV _mepc(R29), R27 - ADD $4, R27 // don't execute syscall instruction again + ADD $4, R27 // Don't execute syscall instruction again MOVV R27, _mepc(R29) - // if fromHandler skip saving thread context + // If fromHandler skip saving thread context AND $1, R27 // fromHandler flag BNE R27, R0, currentStack - // save thread context in mOS - // needs to be done before the syscall, cpuctx.exe might be nil + // Save thread context in mOS + // Needs to be done before the syscall, cpuctx.exe might be nil // afterwards (e.g. in nanosleep) // TODO skip for fast syscall MOVV (cpuctx_exe)(g), R27 @@ -147,7 +147,7 @@ TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 MOVV R26, (m_mOS+mOS_ra)(R27) MOVV _mepc(R29), R26 - AND $~1, R26 // remove fromHandler flag from epc + AND $~1, R26 // Remove fromHandler flag from epc MOVV R26, (m_mOS+mOS_epc)(R27) MOVV (g_sched+gobuf_sp)(g), R26 @@ -165,38 +165,38 @@ duffcopy: // 3 extra registers to preserve src, dst and size of result SUB $sysMaxArgs+3*8, R29 - // copy arguments from the caller's stack + // Copy arguments from the caller's stack MOVV $·duffcopy+2048(SB), R26 SLL $1, R9 SUB R9, R26 MOVV R29, R2 // duffcopy dst JAL (R26) - // save data needed to copy the return values back to the caller's stack + // Save data needed to copy the return values back to the caller's stack MOVV R1, (sysMaxArgs+0*8)(R29) MOVV R2, (sysMaxArgs+1*8)(R29) MOVV R10, (sysMaxArgs+2*8)(R29) - // reenable exceptions + // Reenable exceptions MOVV M(C0_SR), R26 AND $~SR_EXL, R26 MOVV R26, M(C0_SR) - // reminder: don't use R26 or R27 when interrupts enabled + // Reminder: Don't use R26 or R27 when interrupts enabled - // call the service routine + // Call the service routine MOVV $·syscalls(SB), R9 SLL $3, R8 ADD R8, R9 MOVV (R9), R9 JAL (R9) - // disable exceptions again + // Disable exceptions again MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) - // copy the return values back to the caller's stack + // Copy the return values back to the caller's stack MOVV (sysMaxArgs+2*8)(R29), R10 BEQ R0, R10, nothingToCopy MOVV (sysMaxArgs+0*8)(R29), R2 // duffcopy dst @@ -209,19 +209,19 @@ duffcopy: nothingToCopy: ADD $sysMaxArgs+3*8, R29 - // run the scheduler if the syscall wants it - MOVB cpuctx_schedule(g), R8 // handlers will not set this + // Run the scheduler if the syscall wants it + MOVB cpuctx_schedule(g), R8 // Handlers will not set this BEQ R8, R0, 2(PC) JMP ·enterScheduler(SB) - // restore ctx of caller + // Restore ctx of caller MOVV _LR(R29), R26 - AND $~1, R26, R31 // remove smallCtx flag from ra + AND $~1, R26, R31 // Remove smallCtx flag from ra MOVV _mstatus(R29), R26 MOVV R26, M(C0_SR) MOVV _mepc(R29), R26 AND $1, R26, R27 - AND $~1, R26 // remove fromHandler flag from epc + AND $~1, R26 // Remove fromHandler flag from epc MOVV R26, M(C0_EPC) ADD $excCtxSize, R29 @@ -236,23 +236,23 @@ return: ERET -// An interrupt was caused by software. This can happen on -// any instruction. We need to save all GPRs in our context. +// An interrupt was caused by software. This can happen on any instruction. We +// need to save all GPRs in our context. // TODO do we really need to save full context? SW_INT is only set in newwork() TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (cpuctx_exe)(g), R26 MOVV $(m_mOS+mOS_gprs)(R26), R26 JAL ·saveGPRs(SB) - // save thread context in mOS + // Save thread context in mOS MOVV (cpuctx_exe)(g), R27 MOVV _LR(R29), R26 - AND $~1, R26 // remove smallCtx flag from ra + AND $~1, R26 // Remove smallCtx flag from ra MOVV R26, (m_mOS+mOS_ra)(R27) MOVV _mepc(R29), R26 - AND $~1, R26 // remove fromHandler flag from epc + AND $~1, R26 // Remove fromHandler flag from epc MOVV R26, (m_mOS+mOS_epc)(R27) MOVV (g_sched+gobuf_sp)(g), R26 @@ -260,8 +260,8 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (g_sched+gobuf_g)(g), R26 MOVV R26, (m_mOS+mOS_fp)(R27) - // must be a timer or software interrupt. In both cases clear pending - // bits and enter the scheduler + // Must be a timer or software interrupt. In both cases clear pending + // bits and enter the scheduler. MOVV M(C0_CAUSE), R26 AND $~INTR_MASK, R26 MOVV R26, M(C0_CAUSE) @@ -272,41 +272,41 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 - // reenable exceptions + // Reenable exceptions MOVV M(C0_SR), R26 AND $~SR_EXL, R26 // OR $INTR_EXT, R26 // allow external interrupts to preempt scheduler MOVV R26, M(C0_SR) - // reminder: don't use R26 or R27 when interrupts enabled + // Reminder: Don't use R26 or R27 when interrupts enabled - // enter scheduler + // Enter scheduler MOVB R0, cpuctx_schedule(g) JAL ·curcpuRunScheduler(SB) - // disable exceptions again + // Disable exceptions again MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) - // clear cpuctx.newexe + // Clear cpuctx.newexe MOVB R0, (cpuctx_newexe)(g) - // restore mstatus from exception context + // Restore mstatus from exception context MOVV _mstatus(R29), R26 MOVV R26, M(C0_SR) ADD $excCtxSize, R29 MOVV (cpuctx_exe)(g), R27 MOVV (m_mOS+mOS_ra)(R27), R9 - AND $~1, R9, R23 // remove smallCtx flag + AND $~1, R9, R23 // Remove smallCtx flag SUB $8, R29 MOVV R23, 0(R29) AND $1, R9, R10 // smallCtx flag - BNE R0, R10, smallCtx // TODO should never happen + BNE R0, R10, smallCtx // TODO Should never happen MOVV $(m_mOS+mOS_gprs)(R27), R26 - JAL ·restoreGPRs(SB) // TODO skip if we are coming from syscall + JAL ·restoreGPRs(SB) // TODO Skip if we are coming from syscall smallCtx: MOVV 0(R29), R26 @@ -336,7 +336,7 @@ TEXT runtime·externalInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV R27, M(C0_SR) // External interrupts are handled by the application. We need to call - // one of the registered handlers + // one of the registered handlers. MOVV M(C0_CAUSE), R8 AND $INTR_EXT, R8 @@ -344,8 +344,8 @@ TEXT runtime·externalInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV $1, R9 findVector: - // find and handle only the first pending interrupt. If there are - // multiple interrupts pending, the interrupt handler will run again + // Find and handle only the first pending interrupt. If there are + // multiple interrupts pending, the interrupt handler will run again. AND $1, R8, R10 BNE R10, R0, callVector ADD $1, R9 @@ -417,15 +417,13 @@ fatal: JMP ·unhandledExternalInterrupt(SB) -// Required by the linker, runtime.vectors will default to this. +// Do not remove. Required by the linker, runtime.vectors defaults to this. TEXT ·unhandledExternalInterrupt(SB),NOSPLIT|NOFRAME,$0 BREAK JMP -1(PC) -// stolen from runtime/preempt_mips64x.s -// TODO why was temp register (R23) not included in preempt_mips64x? -// R26 must point to where gprs will be stored +// R26 must point to where gprs will be stored. TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 MOVV R1, 0(R26) MOVV R2, 8(R26) @@ -460,9 +458,7 @@ TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 RET -// stolen from runtime/preempt_mips64x.s -// TODO why was temp register (R23) not included in preempt_mips64x? -// R26 must point to stored gprs +// R26 must point to stored gprs. TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 MOVV 216(R26), R1 MOVV R1, LO @@ -497,9 +493,8 @@ TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 RET -// stolen from runtime/preempt_mips64x.s -// might clobber some gprs! -// R26 must point to where fprs will be stored +// Might clobber some gprs! +// R26 must point to where fprs will be stored. TEXT ·saveFPRs(SB),NOSPLIT|NOFRAME,$0 #ifndef GOMIPS64_softfloat MOVV FCR31, R1 @@ -540,9 +535,8 @@ TEXT ·saveFPRs(SB),NOSPLIT|NOFRAME,$0 RET -// stolen from runtime/preempt_mips64x.s -// might clobber some gprs! -// R26 must point to stored fprs +// Might clobber some gprs! +// R26 must point to stored fprs. TEXT ·restoreFPRs(SB),NOSPLIT|NOFRAME,$0 #ifndef GOMIPS64_softfloat MOVD 256(R26), F31 @@ -583,6 +577,8 @@ TEXT ·restoreFPRs(SB),NOSPLIT|NOFRAME,$0 RET +// syscalls not supported by mips64 + // func sysreset(level int, addr unsafe.Pointer) bool TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 NOP // TODO From 08feb8a1347c47427cad7396f1fe28431bf80c44 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 19 Jun 2024 12:07:21 +0200 Subject: [PATCH 38/45] Use only one of the two SW interrupts --- src/internal/cpu/r4000/creg/creg_mips64.go | 8 +++++--- src/runtime/asm_mips64.h | 2 ++ src/runtime/tasker_noos_mips64.go | 7 +++---- src/runtime/tasker_noos_mips64.s | 13 +++++-------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/internal/cpu/r4000/creg/creg_mips64.go b/src/internal/cpu/r4000/creg/creg_mips64.go index 13a740fff27f7d..633bd78e2ef574 100644 --- a/src/internal/cpu/r4000/creg/creg_mips64.go +++ b/src/internal/cpu/r4000/creg/creg_mips64.go @@ -55,9 +55,11 @@ const ( // Cause register bits (p.171) const ( - IP_SW = 0x03 << 8 - IP_EXT = 0x7C << 8 - IP_TIMER = 0x80 << 8 + IP_SW0 = 0x01 << 8 + IP_SW1 = 0x02 << 8 + IP_SWMASK = 0x03 << 8 + IP_EXTMASK = 0x7C << 8 + IP_TIMER = 0x80 << 8 ) func (r CP0Reg) Load() uint32 { return mfc0(r) } diff --git a/src/runtime/asm_mips64.h b/src/runtime/asm_mips64.h index b178402f4ec153..9bf7c97f0e774b 100644 --- a/src/runtime/asm_mips64.h +++ b/src/runtime/asm_mips64.h @@ -64,6 +64,8 @@ // Masks for Status Registers IM bits and Cause Registers IP bits #define INTR_MASK (0xFF << 8) +#define INTR_SW0 (0x01 << 8) +#define INTR_SW1 (0x02 << 8) #define INTR_SW (0x03 << 8) #define INTR_EXT (0x7C << 8) #define INTR_TIMER (0x80 << 8) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index 1d1853c3181cb4..0c676e60ecb998 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -45,14 +45,13 @@ func curcpuSleep() { // when allowing the scheduler to be interrupted. // TODO try again, this might have been my own fault by breaking // atomic operations. - creg.STATUS.SetBits(creg.IP_EXT) - creg.STATUS.ClearBits(creg.IP_EXT) + creg.STATUS.SetBits(creg.IP_EXTMASK) + creg.STATUS.ClearBits(creg.IP_EXTMASK) } //go:nosplit func (cpu *cpuctx) newwork() { - // TODO use only one of the two interrupts - creg.CAUSE.SetBits(creg.IP_SW) + creg.CAUSE.SetBits(creg.IP_SW0) } //go:nosplit diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 47b58df0e30b9e..bfb8c742e38716 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -112,7 +112,7 @@ fromHandler: BEQ R27, R0, 2(PC) JMP ·externalInterruptHandler(SB) - AND $INTR_SW, R26, R27 + AND $INTR_SW0, R26, R27 BEQ R27, R0, 2(PC) JMP ·softwareInterruptHandler(SB) @@ -236,8 +236,8 @@ return: ERET -// An interrupt was caused by software. This can happen on any instruction. We -// need to save all GPRs in our context. +// An interrupt was caused by software, particularly SW0. This can happen on +// any instruction. We need to save all GPRs in our context. // TODO do we really need to save full context? SW_INT is only set in newwork() TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (cpuctx_exe)(g), R26 @@ -260,13 +260,10 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (g_sched+gobuf_g)(g), R26 MOVV R26, (m_mOS+mOS_fp)(R27) - // Must be a timer or software interrupt. In both cases clear pending - // bits and enter the scheduler. + // Clear pending bit and enter the scheduler. MOVV M(C0_CAUSE), R26 - AND $~INTR_MASK, R26 + AND $~INTR_SW0, R26 MOVV R26, M(C0_CAUSE) -// MOVV M(C0_COMPARE), R26 -// MOVV R26, M(C0_COMPARE) JMP ·enterScheduler(SB) From 80103c539f69a54c6b13b99ce2b6834f5c243e50 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 19 Jun 2024 12:21:46 +0200 Subject: [PATCH 39/45] Consistent naming --- src/runtime/tasker_noos_mips64.s | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index bfb8c742e38716..e9de455a8d2399 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -12,7 +12,7 @@ #define sysMaxArgs (48+8) // Exception Context -#define _LR (0*8) +#define _lr (0*8) #define _mstatus (1*8) #define _mepc (2*8) #define excCtxSize (3*8) @@ -81,7 +81,7 @@ fromHandler: // Save exception context on ISR stack SUB $excCtxSize, R29 OR $1, R31, R26 // encode smallCtx flag in ra - MOVV R26, _LR(R29) + MOVV R26, _lr(R29) MOVV M(C0_SR), R26 MOVV R26, _mstatus(R29) MOVV M(C0_EPC), R26 @@ -143,7 +143,7 @@ TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 // TODO skip for fast syscall MOVV (cpuctx_exe)(g), R27 - MOVV _LR(R29), R26 + MOVV _lr(R29), R26 MOVV R26, (m_mOS+mOS_ra)(R27) MOVV _mepc(R29), R26 @@ -215,7 +215,7 @@ nothingToCopy: JMP ·enterScheduler(SB) // Restore ctx of caller - MOVV _LR(R29), R26 + MOVV _lr(R29), R26 AND $~1, R26, R31 // Remove smallCtx flag from ra MOVV _mstatus(R29), R26 MOVV R26, M(C0_SR) @@ -247,7 +247,7 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 // Save thread context in mOS MOVV (cpuctx_exe)(g), R27 - MOVV _LR(R29), R26 + MOVV _lr(R29), R26 AND $~1, R26 // Remove smallCtx flag from ra MOVV R26, (m_mOS+mOS_ra)(R27) @@ -387,7 +387,7 @@ callVector: MOVV _mstatus(R29), R26 MOVV R26, M(C0_SR) - MOVV _LR(R29), R26 + MOVV _lr(R29), R26 MOVV $~1, R27 AND R27, R26, R31 // Remove smallCtx flag from RA MOVV _mepc(R29), R26 From 4bea3f608e7bcb3791033897ae5f685a68cadba0 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 19 Jun 2024 16:34:45 +0200 Subject: [PATCH 40/45] Rework nested exceptions --- src/runtime/tasker_noos_mips64.go | 21 +--- src/runtime/tasker_noos_mips64.s | 171 +++++++++++++++--------------- 2 files changed, 87 insertions(+), 105 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.go b/src/runtime/tasker_noos_mips64.go index 0c676e60ecb998..2db46964caa005 100644 --- a/src/runtime/tasker_noos_mips64.go +++ b/src/runtime/tasker_noos_mips64.go @@ -40,14 +40,7 @@ func taskerinit() { } //go:nosplit -func curcpuSleep() { - // Check defensively for external interrupts here. There were lockups - // when allowing the scheduler to be interrupted. - // TODO try again, this might have been my own fault by breaking - // atomic operations. - creg.STATUS.SetBits(creg.IP_EXTMASK) - creg.STATUS.ClearBits(creg.IP_EXTMASK) -} +func curcpuSleep() {} //go:nosplit func (cpu *cpuctx) newwork() { @@ -55,19 +48,13 @@ func (cpu *cpuctx) newwork() { } //go:nosplit -func curcpuWakeup() { - // TODO implement -} +func curcpuWakeup() {} //go:nosplit -func curcpuSavectxSched() { - // TODO implement -} +func curcpuSavectxSched() {} //go:nosplit -func curcpuSavectxCall() { - // TODO implement -} +func curcpuSavectxCall() {} //go:nosplit func cpuid() int { diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index e9de455a8d2399..b5e869e3fc931d 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -19,7 +19,7 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 - JMP ·exceptionTrap(SB) + JMP ·exceptionHandler(SB) // Main exception handler. // @@ -34,15 +34,15 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 // Called from thread: // 1. Save goroutine context in g.sched (sp, fp) // 2. Switch to ISR stack -// 3. Save exception context on ISR stack (ra, epc, cause) -// 4. Save thread context in m.mOS (sp, fp, ra, epc) +// 3. Save exception context on ISR stack (lr, epc, cause) +// 4. Save thread context in m.mOS (sp, fp, lr, epc) // 5. If interrupt goto 8, if syscall continue // 6. Do syscall // - Copy args from caller thread stack to ISR stack // - Call service routine in ISR context // - Copy result from ISR stack to caller thread stack // 7. If the syscall called curcpuSchedule, make a call to curcpuRunScheduler -// 8. If newexe, restore context from that thread (sp, fp, ra, epc) +// 8. If newexe, restore context from that thread (sp, fp, lr, epc) // 9. Return execution at epc // // Called from handler: @@ -59,7 +59,7 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 // If we want to schedule, context must be in the m object. // - There are no callee save registers in go. Assume all gprs clobbered after // a function call. -TEXT runtime·exceptionTrap(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·exceptionHandler(SB),NOSPLIT|NOFRAME,$0 // Determine caller stack MOVV $·cpu0(SB), R26 BNE R26, g, fromThread @@ -74,23 +74,27 @@ fromThread: MOVV g, (g_sched+gobuf_g)(R26) MOVV (g_stack+stack_hi)(R26), R29 - // Switch to special ISR goroutine + // Switch to "signal" goroutine MOVV R26, g fromHandler: // Save exception context on ISR stack SUB $excCtxSize, R29 - OR $1, R31, R26 // encode smallCtx flag in ra - MOVV R26, _lr(R29) + OR $1, R31, R26 // Encode smallCtx flag in lr + MOVV R26, _lr(R29) // R29 is now free for use MOVV M(C0_SR), R26 MOVV R26, _mstatus(R29) MOVV M(C0_EPC), R26 - OR R27, R26 // encode fromHandler flag in EPC + OR R27, R26 // Encode fromHandler flag in EPC MOVV R26, _mepc(R29) - // Mask interrupts - MOVV M(C0_SR), R26 - MOVV $~(INTR_SW|INTR_EXT), R27 + // Mask pending interrupts. Otherwise they will cause an exception loop + // as soon as we allow nested exceptions. + MOVV M(C0_CAUSE), R26 + MOVV $INTR_MASK, R27 + AND R27, R26 + NOR R26, R26 + MOVV M(C0_SR), R27 AND R27, R26 MOVV R26, M(C0_SR) @@ -129,31 +133,37 @@ fatal: // R9: argument data size on the stack (+8 for caller return address) // R10: return data size on the stack TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 - MOVV _mepc(R29), R27 - ADD $4, R27 // Don't execute syscall instruction again - MOVV R27, _mepc(R29) + // Allow nested exceptions. Reminder: Don't use R26 or R27 when + // exceptions are enabled. + MOVV M(C0_SR), R26 + AND $~SR_EXL, R26 + MOVV R26, M(C0_SR) + + MOVV _mepc(R29), R2 + ADD $4, R2 // Don't execute syscall instruction again + MOVV R2, _mepc(R29) // If fromHandler skip saving thread context - AND $1, R27 // fromHandler flag - BNE R27, R0, currentStack + AND $1, R2 // fromHandler flag + BNE R2, R0, currentStack // Save thread context in mOS // Needs to be done before the syscall, cpuctx.exe might be nil // afterwards (e.g. in nanosleep) // TODO skip for fast syscall - MOVV (cpuctx_exe)(g), R27 + MOVV (cpuctx_exe)(g), R2 - MOVV _lr(R29), R26 - MOVV R26, (m_mOS+mOS_ra)(R27) + MOVV _lr(R29), R1 + MOVV R1, (m_mOS+mOS_ra)(R2) - MOVV _mepc(R29), R26 - AND $~1, R26 // Remove fromHandler flag from epc - MOVV R26, (m_mOS+mOS_epc)(R27) + MOVV _mepc(R29), R1 + AND $~1, R1 // Remove fromHandler flag from epc + MOVV R1, (m_mOS+mOS_epc)(R2) - MOVV (g_sched+gobuf_sp)(g), R26 - MOVV R26, (m_mOS+mOS_sp)(R27) - MOVV (g_sched+gobuf_g)(g), R26 - MOVV R26, (m_mOS+mOS_fp)(R27) + MOVV (g_sched+gobuf_sp)(g), R1 + MOVV R1, (m_mOS+mOS_sp)(R2) + MOVV (g_sched+gobuf_g)(g), R1 + MOVV R1, (m_mOS+mOS_fp)(R2) MOVV (g_sched+gobuf_sp)(g), R1 // duffcopy src thread JMP duffcopy @@ -166,24 +176,17 @@ duffcopy: SUB $sysMaxArgs+3*8, R29 // Copy arguments from the caller's stack - MOVV $·duffcopy+2048(SB), R26 + MOVV $·duffcopy+2048(SB), R3 SLL $1, R9 - SUB R9, R26 + SUB R9, R3 MOVV R29, R2 // duffcopy dst - JAL (R26) + JAL (R3) // Save data needed to copy the return values back to the caller's stack MOVV R1, (sysMaxArgs+0*8)(R29) MOVV R2, (sysMaxArgs+1*8)(R29) MOVV R10, (sysMaxArgs+2*8)(R29) - // Reenable exceptions - MOVV M(C0_SR), R26 - AND $~SR_EXL, R26 - MOVV R26, M(C0_SR) - - // Reminder: Don't use R26 or R27 when interrupts enabled - // Call the service routine MOVV $·syscalls(SB), R9 SLL $3, R8 @@ -191,20 +194,15 @@ duffcopy: MOVV (R9), R9 JAL (R9) - // Disable exceptions again - MOVV M(C0_SR), R8 - OR $SR_EXL, R8 - MOVV R8, M(C0_SR) - // Copy the return values back to the caller's stack MOVV (sysMaxArgs+2*8)(R29), R10 BEQ R0, R10, nothingToCopy MOVV (sysMaxArgs+0*8)(R29), R2 // duffcopy dst MOVV (sysMaxArgs+1*8)(R29), R1 // duffcopy src - MOVV $·duffcopy+2048(SB), R26 + MOVV $·duffcopy+2048(SB), R3 SLL $1, R10 - SUB R10, R26 - JAL (R26) + SUB R10, R3 + JAL (R3) nothingToCopy: ADD $sysMaxArgs+3*8, R29 @@ -214,23 +212,28 @@ nothingToCopy: BEQ R8, R0, 2(PC) JMP ·enterScheduler(SB) + // Disable nested exceptions + MOVV M(C0_SR), R8 + OR $SR_EXL, R8 + MOVV R8, M(C0_SR) + // Restore ctx of caller - MOVV _lr(R29), R26 - AND $~1, R26, R31 // Remove smallCtx flag from ra - MOVV _mstatus(R29), R26 - MOVV R26, M(C0_SR) - MOVV _mepc(R29), R26 - AND $1, R26, R27 - AND $~1, R26 // Remove fromHandler flag from epc - MOVV R26, M(C0_EPC) + MOVV _lr(R29), R1 + AND $~1, R1, R31 // Remove smallCtx flag from lr + MOVV _mstatus(R29), R1 + MOVV R1, M(C0_SR) + MOVV _mepc(R29), R1 + AND $1, R1, R2 + AND $~1, R1 // Remove fromHandler flag from epc + MOVV R1, M(C0_EPC) ADD $excCtxSize, R29 - BNE R27, R0, return + BNE R2, R0, return - MOVV $·cpu0(SB), R26 - MOVV (g_sched+gobuf_sp)(R26), R29 - MOVV (g_sched+gobuf_g)(R26), g + MOVV $·cpu0(SB), R1 + MOVV (g_sched+gobuf_sp)(R1), R29 + MOVV (g_sched+gobuf_g)(R1), g return: ERET @@ -248,7 +251,7 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (cpuctx_exe)(g), R27 MOVV _lr(R29), R26 - AND $~1, R26 // Remove smallCtx flag from ra + AND $~1, R26 // Remove smallCtx flag from lr MOVV R26, (m_mOS+mOS_ra)(R27) MOVV _mepc(R29), R26 @@ -260,6 +263,11 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (g_sched+gobuf_g)(g), R26 MOVV R26, (m_mOS+mOS_fp)(R27) + // Enable nested exceptions + MOVV M(C0_SR), R26 + AND $~SR_EXL, R26 + MOVV R26, M(C0_SR) + // Clear pending bit and enter the scheduler. MOVV M(C0_CAUSE), R26 AND $~INTR_SW0, R26 @@ -268,20 +276,14 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 JMP ·enterScheduler(SB) +// Reminder: At this point nested exceptions should always be enabled. +// Don't use R26 or R27 when exceptions enabled. TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 - // Reenable exceptions - MOVV M(C0_SR), R26 - AND $~SR_EXL, R26 -// OR $INTR_EXT, R26 // allow external interrupts to preempt scheduler - MOVV R26, M(C0_SR) - - // Reminder: Don't use R26 or R27 when interrupts enabled - // Enter scheduler MOVB R0, cpuctx_schedule(g) JAL ·curcpuRunScheduler(SB) - // Disable exceptions again + // Disable nested exceptions MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) @@ -300,10 +302,10 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 SUB $8, R29 MOVV R23, 0(R29) AND $1, R9, R10 // smallCtx flag - BNE R0, R10, smallCtx // TODO Should never happen + BNE R0, R10, smallCtx MOVV $(m_mOS+mOS_gprs)(R27), R26 - JAL ·restoreGPRs(SB) // TODO Skip if we are coming from syscall + JAL ·restoreGPRs(SB) smallCtx: MOVV 0(R29), R26 @@ -319,15 +321,17 @@ smallCtx: TEXT runtime·externalInterruptHandler(SB),NOSPLIT|NOFRAME,$0 // External interrupt can happen anytime, save full context. Context - // will be saved on the stack, to allow nested interrupts be supported. - // TODO nested interrupts aren't enabled as of now. Do we need them? + // will be saved on the ISR stack, to allow nested interrupts be + // supported. + // Keep in mind that the scheduler must not be called if context is - // saved on the stack. + // saved on the stack. TODO why? + SUB $const_numGPRS*8, R29 MOVV R29, R26 JAL ·saveGPRs(SB) - // Context is saved. Reenable exceptions. Don't use R26, R27. + // Context is saved. Enable nested exceptions. Don't use R26, R27. MOVV M(C0_SR), R27 AND $~SR_EXL, R27 MOVV R27, M(C0_SR) @@ -360,27 +364,18 @@ callVector: ADD R8, R10 MOVV (R10), R10 - // Allow nested interrupts -// MOVV M(C0_SR), R8 -// OR $INTR_SW, R8 -// MOVV R8, M(C0_SR) - JAL (R10) -// MOVV M(C0_SR), R8 -// AND $~INTR_SW, R8 -// MOVV R8, M(C0_SR) - // External interrupts can't be cleared by clearing the bits in cause // register. The user handler must have cleared the interrupt at this - // point in the external source, probably via mmio. + // point in the external source. - // Disable exceptions again. Restore Context. + // Disable nested exceptions again MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) - // Restore ctx of caller + // Restore context of caller MOVV R29, R26 JAL ·restoreGPRs(SB) ADD $const_numGPRS*8, R29 @@ -389,7 +384,7 @@ callVector: MOVV R26, M(C0_SR) MOVV _lr(R29), R26 MOVV $~1, R27 - AND R27, R26, R31 // Remove smallCtx flag from RA + AND R27, R26, R31 // Remove smallCtx flag from lr MOVV _mepc(R29), R26 MOVV $~1, R27 AND R26, R27 // Remove fromHandler flag from EPC @@ -578,9 +573,9 @@ TEXT ·restoreFPRs(SB),NOSPLIT|NOFRAME,$0 // func sysreset(level int, addr unsafe.Pointer) bool TEXT ·sysreset(SB),NOSPLIT|NOFRAME,$0-12 - NOP // TODO + NOP // func syscachemaint(op int, p unsafe.Pointer, size int) TEXT ·syscachemaint(SB),NOSPLIT,$0-12 - NOP // TODO + NOP From f31c852e26784559a56df086137c6dcf29ba197f Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Wed, 19 Jun 2024 17:01:06 +0200 Subject: [PATCH 41/45] Handle fast syscalls properly --- src/runtime/tasker_noos_mips64.s | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index b5e869e3fc931d..5b39554bff8379 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -143,32 +143,37 @@ TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 ADD $4, R2 // Don't execute syscall instruction again MOVV R2, _mepc(R29) + MOVV $SYS_LAST_FAST, R4 + SUB R4, R8, R4 // R4 <= 0: fast syscall + // If fromHandler skip saving thread context AND $1, R2 // fromHandler flag BNE R2, R0, currentStack + MOVV (g_sched+gobuf_sp)(g), R1 // duffcopy src thread + BLEZ R4, duffcopy // fast syscall + // Save thread context in mOS // Needs to be done before the syscall, cpuctx.exe might be nil // afterwards (e.g. in nanosleep) - // TODO skip for fast syscall MOVV (cpuctx_exe)(g), R2 - MOVV _lr(R29), R1 - MOVV R1, (m_mOS+mOS_ra)(R2) + MOVV _lr(R29), R3 + MOVV R3, (m_mOS+mOS_ra)(R2) - MOVV _mepc(R29), R1 - AND $~1, R1 // Remove fromHandler flag from epc - MOVV R1, (m_mOS+mOS_epc)(R2) + MOVV _mepc(R29), R3 + AND $~1, R3 // Remove fromHandler flag from epc + MOVV R3, (m_mOS+mOS_epc)(R2) - MOVV (g_sched+gobuf_sp)(g), R1 - MOVV R1, (m_mOS+mOS_sp)(R2) - MOVV (g_sched+gobuf_g)(g), R1 - MOVV R1, (m_mOS+mOS_fp)(R2) + MOVV (g_sched+gobuf_sp)(g), R3 + MOVV R3, (m_mOS+mOS_sp)(R2) + MOVV (g_sched+gobuf_g)(g), R3 + MOVV R3, (m_mOS+mOS_fp)(R2) - MOVV (g_sched+gobuf_sp)(g), R1 // duffcopy src thread JMP duffcopy currentStack: + BGTZ R4, badSyscall // slow syscall from handler ADD $excCtxSize, R29, R1 // duffcopy src handler duffcopy: @@ -238,6 +243,9 @@ nothingToCopy: return: ERET +badSyscall: + BREAK + // An interrupt was caused by software, particularly SW0. This can happen on // any instruction. We need to save all GPRs in our context. From 6c2b71d91cc68b5bf2a56ac80132cd82c8d3105a Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 20 Jun 2024 07:08:08 +0200 Subject: [PATCH 42/45] Improve documentation --- src/runtime/tasker_noos_mips64.s | 128 ++++++++++++++++++------------- 1 file changed, 74 insertions(+), 54 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index 5b39554bff8379..c3a7c19a80976a 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -2,6 +2,37 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Implementation of the exception handler for the tasker. +// +// There are different levels of context that we save on preemption: +// - goroutine (sp, fp): Needed to return to the preempted goroutine +// from the "signal" goroutine, if we didn't schedule. Saved in g.sched. +// - exception (lr, status, epc): Registers changed by the exception, must be +// restored at the end of the exception. Saved on "signal" stack, i.e. might +// be nested. +// - thread (sp, fp, lr, epc): Needed to schedule a goroutine. If we are +// going to call the scheduler, the goroutine context is moved into the thread +// context. Saved in m.mOS. +// - full (gprs, fprs): Full context saves all registers. Needed on arbitrary +// preemption, i.e. interrupts. These will be saved on the "signal" stack or +// in m.mOS. External interrupts will choose the former, to allow nested +// interrupts. Software interrupts will choose the latter, to allow +// scheduling. +// +// There are some conditions that affect how the exception is handled: +// - fromHandler: This flag is set when the exception preempted the "signal" +// goroutine, i.e. we are in a nested exception. A nested exception must not +// enter the scheduler, but always return to the caller stack. Therefore it's +// also only allowed to call fast syscalls, which are guaranteed to not call +// the scheduler. This flag is encoded in bit 0 of the exception context's +// _mepc value. +// - smallCtx: This flag indicates that we didn't save the gprs/fprs. We don't +// need to save them if preemption happened at function entry, i.e. syscalls. +// The caller won't care about these because of the Go ABI0. This flag is +// encoded in bit 0 of the exception context's _lr value. +// - fast syscall: Fast syscalls are guaranteed to not enter the scheduler. We +// need only to save the goroutine's and exception's context. + #include "go_asm.h" #include "go_tls.h" #include "textflag.h" @@ -18,10 +49,17 @@ #define excCtxSize (3*8) +// This will be copied into the processor's general exception vector. Since the +// general exception vector is limited in size, it just jumps to the actual +// implementation. TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 JMP ·exceptionHandler(SB) -// Main exception handler. + +// Main exception handler. If necessary saves goroutine context and switches +// stach to "signal" goroutine. Saves exceptions context. Masks pending +// interrupts in preparation for nested interrupts. Lastly evaluates the +// exception cause and calls the appropriate function. // // R26 and R27 are free, see runtime/asm_mips64x.s // @@ -30,35 +68,6 @@ TEXT runtime·intvector(SB),NOSPLIT|NOFRAME,$0 // // Only syscalls and interrupts are handled at the moment, all other exceptions // are fatal. -// -// Called from thread: -// 1. Save goroutine context in g.sched (sp, fp) -// 2. Switch to ISR stack -// 3. Save exception context on ISR stack (lr, epc, cause) -// 4. Save thread context in m.mOS (sp, fp, lr, epc) -// 5. If interrupt goto 8, if syscall continue -// 6. Do syscall -// - Copy args from caller thread stack to ISR stack -// - Call service routine in ISR context -// - Copy result from ISR stack to caller thread stack -// 7. If the syscall called curcpuSchedule, make a call to curcpuRunScheduler -// 8. If newexe, restore context from that thread (sp, fp, lr, epc) -// 9. Return execution at epc -// -// Called from handler: -// same, but skip 1, 2, 4 -// -// --external-------ERET -// / -// exception---interrupt--SW----scheduler--ERET -// \ / -// --syscall------ -// -// Other basic things I learned while writing this: -// - If we want to support nested exceptions, save context must be on the stack. -// If we want to schedule, context must be in the m object. -// - There are no callee save registers in go. Assume all gprs clobbered after -// a function call. TEXT runtime·exceptionHandler(SB),NOSPLIT|NOFRAME,$0 // Determine caller stack MOVV $·cpu0(SB), R26 @@ -89,7 +98,7 @@ fromHandler: MOVV R26, _mepc(R29) // Mask pending interrupts. Otherwise they will cause an exception loop - // as soon as we allow nested exceptions. + // as soon as we allow nested interrupts. MOVV M(C0_CAUSE), R26 MOVV $INTR_MASK, R27 AND R27, R26 @@ -124,16 +133,24 @@ fatal: JMP ·unhandledExcepton_target(SB) -// System call is like oridnary function call so all registers are caller save -// (Go ABI0). Meaning we don't need to save GPRs and are free to use them. The -// tiny wrapper over SYSCALL instruction adds additional parameters in R8-R10 -// registers: +// Execute a system call. First unsets exception level to allow nested +// interrupts. Forwards epc by one instruction, otherwise the syscall is +// triggered again on ERET. Saves thread context if we might schedule, i.e. not +// a fast syscall and not nested exception. Duffcopies the syscall arguments +// from caller stack to "signal" stack. Jump to syscall handler. Duffcopy +// results back. It the syscall called curcpuSchedule, jump to enterScheduler. +// Otherwise restore exception and goroutine context. +// +// System call is like an oridnary function call, so all registers are caller +// save (Go ABI0). Meaning we don't need to save GPRs and are free to use them. +// The tiny wrapper over SYSCALL instruction adds additional parameters in +// R8-R10 registers: // // R8: syscall number // R9: argument data size on the stack (+8 for caller return address) // R10: return data size on the stack TEXT runtime·syscallHandler(SB),NOSPLIT|NOFRAME,$0 - // Allow nested exceptions. Reminder: Don't use R26 or R27 when + // Allow nested interrupts. Reminder: Don't use R26 or R27 when // exceptions are enabled. MOVV M(C0_SR), R26 AND $~SR_EXL, R26 @@ -213,11 +230,11 @@ nothingToCopy: ADD $sysMaxArgs+3*8, R29 // Run the scheduler if the syscall wants it - MOVB cpuctx_schedule(g), R8 // Handlers will not set this + MOVB cpuctx_schedule(g), R8 // Always false if nested or fast syscall BEQ R8, R0, 2(PC) JMP ·enterScheduler(SB) - // Disable nested exceptions + // Disable nested interrupts MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) @@ -248,7 +265,9 @@ badSyscall: // An interrupt was caused by software, particularly SW0. This can happen on -// any instruction. We need to save all GPRs in our context. +// any instruction. Saves full context in the thread context, enables nested +// interrupts and jumps to enterScheduler. +// // TODO do we really need to save full context? SW_INT is only set in newwork() TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (cpuctx_exe)(g), R26 @@ -271,7 +290,7 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 MOVV (g_sched+gobuf_g)(g), R26 MOVV R26, (m_mOS+mOS_fp)(R27) - // Enable nested exceptions + // Enable nested interrupts MOVV M(C0_SR), R26 AND $~SR_EXL, R26 MOVV R26, M(C0_SR) @@ -284,14 +303,14 @@ TEXT runtime·softwareInterruptHandler(SB),NOSPLIT|NOFRAME,$0 JMP ·enterScheduler(SB) -// Reminder: At this point nested exceptions should always be enabled. -// Don't use R26 or R27 when exceptions enabled. +// Calls the scheduler and switches to the newly scheduled goroutine. Assumes +// nested interrupts are enabled. TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 // Enter scheduler MOVB R0, cpuctx_schedule(g) JAL ·curcpuRunScheduler(SB) - // Disable nested exceptions + // Disable nested interrupts MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) @@ -314,6 +333,7 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 MOVV $(m_mOS+mOS_gprs)(R27), R26 JAL ·restoreGPRs(SB) + // Only use R26, R27 from here smallCtx: MOVV 0(R29), R26 @@ -327,19 +347,18 @@ smallCtx: ERET +// Calls a user implementation for a pending external interrupt. External +// interrupt can happen anytime, saves full context. Context will be saved on +// the ISR stack, to allow nested interrupts be supported. Calculates the +// runtime.vector offset from the interrupt number and calls it. Restores +// context and returns. +// TODO save fprs? TEXT runtime·externalInterruptHandler(SB),NOSPLIT|NOFRAME,$0 - // External interrupt can happen anytime, save full context. Context - // will be saved on the ISR stack, to allow nested interrupts be - // supported. - - // Keep in mind that the scheduler must not be called if context is - // saved on the stack. TODO why? - SUB $const_numGPRS*8, R29 MOVV R29, R26 JAL ·saveGPRs(SB) - // Context is saved. Enable nested exceptions. Don't use R26, R27. + // Context is saved. Enable nested interrupts. Don't use R26, R27. MOVV M(C0_SR), R27 AND $~SR_EXL, R27 MOVV R27, M(C0_SR) @@ -364,7 +383,7 @@ findVector: callVector: SLL $3, R9, R8 // irq vector offset - // get interrupt vector + // Get interrupt vector MOVV $runtime·vectors(SB), R10 MOVV (R10), R11 SUB R9, R11 @@ -378,7 +397,7 @@ callVector: // register. The user handler must have cleared the interrupt at this // point in the external source. - // Disable nested exceptions again + // Disable nested interrupts MOVV M(C0_SR), R8 OR $SR_EXL, R8 MOVV R8, M(C0_SR) @@ -386,6 +405,7 @@ callVector: // Restore context of caller MOVV R29, R26 JAL ·restoreGPRs(SB) + // Only use R26, R27 from here ADD $const_numGPRS*8, R29 MOVV _mstatus(R29), R26 @@ -458,7 +478,7 @@ TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 RET -// R26 must point to stored gprs. +// R26 must point to stored gprs. Only use R26, R27 after restoring. TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 MOVV 216(R26), R1 MOVV R1, LO From c13d8c9b794ce1f8a99a673e961ab77d08e0dcae Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 20 Jun 2024 08:54:33 +0200 Subject: [PATCH 43/45] Make context restoration preemptable if possbile --- src/runtime/tasker_noos_mips64.s | 37 ++++++++++++++------------------ 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index c3a7c19a80976a..b49b6fa3d458f3 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -234,11 +234,6 @@ nothingToCopy: BEQ R8, R0, 2(PC) JMP ·enterScheduler(SB) - // Disable nested interrupts - MOVV M(C0_SR), R8 - OR $SR_EXL, R8 - MOVV R8, M(C0_SR) - // Restore ctx of caller MOVV _lr(R29), R1 AND $~1, R1, R31 // Remove smallCtx flag from lr @@ -310,39 +305,37 @@ TEXT runtime·enterScheduler(SB),NOSPLIT|NOFRAME,$0 MOVB R0, cpuctx_schedule(g) JAL ·curcpuRunScheduler(SB) - // Disable nested interrupts - MOVV M(C0_SR), R8 - OR $SR_EXL, R8 - MOVV R8, M(C0_SR) - // Clear cpuctx.newexe MOVB R0, (cpuctx_newexe)(g) // Restore mstatus from exception context - MOVV _mstatus(R29), R26 - MOVV R26, M(C0_SR) + MOVV _mstatus(R29), R1 + MOVV R1, M(C0_SR) ADD $excCtxSize, R29 + // Disable nested interrupts + MOVV M(C0_SR), R8 + OR $SR_EXL, R8 + MOVV R8, M(C0_SR) + MOVV (cpuctx_exe)(g), R27 MOVV (m_mOS+mOS_ra)(R27), R9 - AND $~1, R9, R23 // Remove smallCtx flag - SUB $8, R29 - MOVV R23, 0(R29) - AND $1, R9, R10 // smallCtx flag - BNE R0, R10, smallCtx + + AND $1, R9 // smallCtx flag + BNE R0, R9, smallCtx MOVV $(m_mOS+mOS_gprs)(R27), R26 JAL ·restoreGPRs(SB) // Only use R26, R27 from here smallCtx: - MOVV 0(R29), R26 - ADD $8, R29 MOVV (m_mOS+mOS_sp)(R27), R29 MOVV (m_mOS+mOS_fp)(R27), g - MOVV R26, R31 + MOVV (m_mOS+mOS_ra)(R27), R31 MOVV (m_mOS+mOS_epc)(R27), R26 MOVV R26, M(C0_EPC) + MOVV $~1, R27 + AND R27, R31 // Remove smallCtx flag ERET @@ -478,7 +471,9 @@ TEXT ·saveGPRs(SB),NOSPLIT|NOFRAME,$0 RET -// R26 must point to stored gprs. Only use R26, R27 after restoring. +// R26 must point to stored gprs. Only use R26, R27 after restoring. Be +// especially careful and look at the disassembly; The assembler might decide +// to use R16-R23 for you. TEXT ·restoreGPRs(SB),NOSPLIT|NOFRAME,$0 MOVV 216(R26), R1 MOVV R1, LO From d4e2e31e7e30881660dbd6c2158d81e1ca5cd966 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 20 Jun 2024 09:09:22 +0200 Subject: [PATCH 44/45] Remove unreachable code --- src/runtime/tasker_noos_mips64.s | 1 - 1 file changed, 1 deletion(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index b49b6fa3d458f3..d6a460893320fa 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -433,7 +433,6 @@ fatal: // Do not remove. Required by the linker, runtime.vectors defaults to this. TEXT ·unhandledExternalInterrupt(SB),NOSPLIT|NOFRAME,$0 BREAK - JMP -1(PC) // R26 must point to where gprs will be stored. From be0dfbbff184a1bbe9c18b81349cbda488cc8543 Mon Sep 17 00:00:00 2001 From: Timur Celik Date: Thu, 20 Jun 2024 10:22:25 +0200 Subject: [PATCH 45/45] Allow nesting only for external interrupts The implementation of softwareInterruptHandler can't support nesting, as it must call the scheduler. --- src/runtime/tasker_noos_mips64.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/tasker_noos_mips64.s b/src/runtime/tasker_noos_mips64.s index d6a460893320fa..dcc9e7d36707a5 100644 --- a/src/runtime/tasker_noos_mips64.s +++ b/src/runtime/tasker_noos_mips64.s @@ -100,7 +100,7 @@ fromHandler: // Mask pending interrupts. Otherwise they will cause an exception loop // as soon as we allow nested interrupts. MOVV M(C0_CAUSE), R26 - MOVV $INTR_MASK, R27 + MOVV $INTR_EXT, R27 // nesteding only allowed for external interrupts AND R27, R26 NOR R26, R26 MOVV M(C0_SR), R27