240127.4 · 0b9a92418f4b39f85888e0704118fed7c2d36fe9
Improved performance by using switch statements instead of maps for state transitions.
reading declarations
@@ -39,11 +39,10 @@ const ( // // Unicode version 15.0.0. func grTransitions(state, prop int) (newState int, newProp int, boundary int) {- // It turns out that using a big switch statement is faster than using a map- // by a factor of about 2.3.+ // It turns out that using a big switch statement is much faster than using+ // a map. - key := uint64(state) | uint64(prop)<<32- switch key {+ switch uint64(state) | uint64(prop)<<32 { // GB5 case grAny | prCR<<32: return grCR, grBoundary, 50@@ -139,10 +138,10 @@ func transitionGraphemeState(state int, r rune) (newState, prop int, boundary bo prop = propertyGraphemes(r) // Find the applicable transition.- transState, transProp, _ := grTransitions(state, prop)- if transState >= 0 {+ nextState, nextProp, _ := grTransitions(state, prop)+ if nextState >= 0 { // We have a specific transition. We'll use it.- return transState, prop, transProp == grBoundary+ return nextState, prop, nextProp == grBoundary } // No specific transition found. Try the less specific ones.
@@ -122,13 +122,13 @@ func FirstLineSegmentInString(str string, state int) (segment, rest string, must // [UAX #14]: https://www.unicode.org/reports/tr14/#Algorithm func HasTrailingLineBreak(b []byte) bool { r, _ := utf8.DecodeLastRune(b)- property, _ := propertyWithGenCat(lineBreakCodePoints, r)+ property, _ := propertyLineBreak(r) return property == prBK || property == prCR || property == prLF || property == prNL } // HasTrailingLineBreakInString is like [HasTrailingLineBreak] but for a string. func HasTrailingLineBreakInString(str string) bool { r, _ := utf8.DecodeLastRuneInString(str)- property, _ := propertyWithGenCat(lineBreakCodePoints, r)+ property, _ := propertyLineBreak(r) return property == prBK || property == prCR || property == prLF || property == prNL }
@@ -64,222 +64,381 @@ const ( LineMustBreak // You must break the line here. ) -// The line break parser's state transitions. It's anologous to grTransitions,-// see comments there for details. Unicode version 15.0.0.-var lbTransitions = map[[2]int][3]int{+// lbTransitions implements the line break parser's state transitions. It's+// anologous to [grTransitions], see comments there for details.+//+// Unicode version 15.0.0.+func lbTransitions(state, prop int) (newState, lineBreak, rule int) {+ switch uint64(state) | uint64(prop)<<32 { // LB4.- {lbAny, prBK}: {lbBK, LineCanBreak, 310},- {lbBK, prAny}: {lbAny, LineMustBreak, 40},+ case lbBK | prAny<<32:+ return lbAny, LineMustBreak, 40 // LB5.- {lbAny, prCR}: {lbCR, LineCanBreak, 310},- {lbAny, prLF}: {lbLF, LineCanBreak, 310},- {lbAny, prNL}: {lbNL, LineCanBreak, 310},- {lbCR, prLF}: {lbLF, LineDontBreak, 50},- {lbCR, prAny}: {lbAny, LineMustBreak, 50},- {lbLF, prAny}: {lbAny, LineMustBreak, 50},- {lbNL, prAny}: {lbAny, LineMustBreak, 50},+ case lbCR | prLF<<32:+ return lbLF, LineDontBreak, 50+ case lbCR | prAny<<32:+ return lbAny, LineMustBreak, 50+ case lbLF | prAny<<32:+ return lbAny, LineMustBreak, 50+ case lbNL | prAny<<32:+ return lbAny, LineMustBreak, 50 // LB6.- {lbAny, prBK}: {lbBK, LineDontBreak, 60},- {lbAny, prCR}: {lbCR, LineDontBreak, 60},- {lbAny, prLF}: {lbLF, LineDontBreak, 60},- {lbAny, prNL}: {lbNL, LineDontBreak, 60},+ case lbAny | prBK<<32:+ return lbBK, LineDontBreak, 60+ case lbAny | prCR<<32:+ return lbCR, LineDontBreak, 60+ case lbAny | prLF<<32:+ return lbLF, LineDontBreak, 60+ case lbAny | prNL<<32:+ return lbNL, LineDontBreak, 60 // LB7.- {lbAny, prSP}: {lbSP, LineDontBreak, 70},- {lbAny, prZW}: {lbZW, LineDontBreak, 70},+ case lbAny | prSP<<32:+ return lbSP, LineDontBreak, 70+ case lbAny | prZW<<32:+ return lbZW, LineDontBreak, 70 // LB8.- {lbZW, prSP}: {lbZW, LineDontBreak, 70},- {lbZW, prAny}: {lbAny, LineCanBreak, 80},+ case lbZW | prSP<<32:+ return lbZW, LineDontBreak, 70+ case lbZW | prAny<<32:+ return lbAny, LineCanBreak, 80 // LB11.- {lbAny, prWJ}: {lbWJ, LineDontBreak, 110},- {lbWJ, prAny}: {lbAny, LineDontBreak, 110},+ case lbAny | prWJ<<32:+ return lbWJ, LineDontBreak, 110+ case lbWJ | prAny<<32:+ return lbAny, LineDontBreak, 110 // LB12.- {lbAny, prGL}: {lbGL, LineCanBreak, 310},- {lbGL, prAny}: {lbAny, LineDontBreak, 120},+ case lbAny | prGL<<32:+ return lbGL, LineCanBreak, 310+ case lbGL | prAny<<32:+ return lbAny, LineDontBreak, 120 // LB13 (simple transitions).- {lbAny, prCL}: {lbCL, LineCanBreak, 310},- {lbAny, prCP}: {lbCP, LineCanBreak, 310},- {lbAny, prEX}: {lbEX, LineDontBreak, 130},- {lbAny, prIS}: {lbIS, LineCanBreak, 310},- {lbAny, prSY}: {lbSY, LineCanBreak, 310},+ case lbAny | prCL<<32:+ return lbCL, LineCanBreak, 310+ case lbAny | prCP<<32:+ return lbCP, LineCanBreak, 310+ case lbAny | prEX<<32:+ return lbEX, LineDontBreak, 130+ case lbAny | prIS<<32:+ return lbIS, LineCanBreak, 310+ case lbAny | prSY<<32:+ return lbSY, LineCanBreak, 310 // LB14.- {lbAny, prOP}: {lbOP, LineCanBreak, 310},- {lbOP, prSP}: {lbOP, LineDontBreak, 70},- {lbOP, prAny}: {lbAny, LineDontBreak, 140},+ case lbAny | prOP<<32:+ return lbOP, LineCanBreak, 310+ case lbOP | prSP<<32:+ return lbOP, LineDontBreak, 70+ case lbOP | prAny<<32:+ return lbAny, LineDontBreak, 140 // LB15.- {lbQU, prSP}: {lbQUSP, LineDontBreak, 70},- {lbQU, prOP}: {lbOP, LineDontBreak, 150},- {lbQUSP, prOP}: {lbOP, LineDontBreak, 150},+ case lbQU | prSP<<32:+ return lbQUSP, LineDontBreak, 70+ case lbQU | prOP<<32:+ return lbOP, LineDontBreak, 150+ case lbQUSP | prOP<<32:+ return lbOP, LineDontBreak, 150 // LB16.- {lbCL, prSP}: {lbCLCPSP, LineDontBreak, 70},- {lbNUCL, prSP}: {lbCLCPSP, LineDontBreak, 70},- {lbCP, prSP}: {lbCLCPSP, LineDontBreak, 70},- {lbNUCP, prSP}: {lbCLCPSP, LineDontBreak, 70},- {lbCL, prNS}: {lbNS, LineDontBreak, 160},- {lbNUCL, prNS}: {lbNS, LineDontBreak, 160},- {lbCP, prNS}: {lbNS, LineDontBreak, 160},- {lbNUCP, prNS}: {lbNS, LineDontBreak, 160},- {lbCLCPSP, prNS}: {lbNS, LineDontBreak, 160},+ case lbCL | prSP<<32:+ return lbCLCPSP, LineDontBreak, 70+ case lbNUCL | prSP<<32:+ return lbCLCPSP, LineDontBreak, 70+ case lbCP | prSP<<32:+ return lbCLCPSP, LineDontBreak, 70+ case lbNUCP | prSP<<32:+ return lbCLCPSP, LineDontBreak, 70+ case lbCL | prNS<<32:+ return lbNS, LineDontBreak, 160+ case lbNUCL | prNS<<32:+ return lbNS, LineDontBreak, 160+ case lbCP | prNS<<32:+ return lbNS, LineDontBreak, 160+ case lbNUCP | prNS<<32:+ return lbNS, LineDontBreak, 160+ case lbCLCPSP | prNS<<32:+ return lbNS, LineDontBreak, 160 // LB17.- {lbAny, prB2}: {lbB2, LineCanBreak, 310},- {lbB2, prSP}: {lbB2SP, LineDontBreak, 70},- {lbB2, prB2}: {lbB2, LineDontBreak, 170},- {lbB2SP, prB2}: {lbB2, LineDontBreak, 170},+ case lbAny | prB2<<32:+ return lbB2, LineCanBreak, 310+ case lbB2 | prSP<<32:+ return lbB2SP, LineDontBreak, 70+ case lbB2 | prB2<<32:+ return lbB2, LineDontBreak, 170+ case lbB2SP | prB2<<32:+ return lbB2, LineDontBreak, 170 // LB18.- {lbSP, prAny}: {lbAny, LineCanBreak, 180},- {lbQUSP, prAny}: {lbAny, LineCanBreak, 180},- {lbCLCPSP, prAny}: {lbAny, LineCanBreak, 180},- {lbB2SP, prAny}: {lbAny, LineCanBreak, 180},+ case lbSP | prAny<<32:+ return lbAny, LineCanBreak, 180+ case lbQUSP | prAny<<32:+ return lbAny, LineCanBreak, 180+ case lbCLCPSP | prAny<<32:+ return lbAny, LineCanBreak, 180+ case lbB2SP | prAny<<32:+ return lbAny, LineCanBreak, 180 // LB19.- {lbAny, prQU}: {lbQU, LineDontBreak, 190},- {lbQU, prAny}: {lbAny, LineDontBreak, 190},+ case lbAny | prQU<<32:+ return lbQU, LineDontBreak, 190+ case lbQU | prAny<<32:+ return lbAny, LineDontBreak, 190 // LB20.- {lbAny, prCB}: {lbCB, LineCanBreak, 200},- {lbCB, prAny}: {lbAny, LineCanBreak, 200},+ case lbAny | prCB<<32:+ return lbCB, LineCanBreak, 200+ case lbCB | prAny<<32:+ return lbAny, LineCanBreak, 200 // LB21.- {lbAny, prBA}: {lbBA, LineDontBreak, 210},- {lbAny, prHY}: {lbHY, LineDontBreak, 210},- {lbAny, prNS}: {lbNS, LineDontBreak, 210},- {lbAny, prBB}: {lbBB, LineCanBreak, 310},- {lbBB, prAny}: {lbAny, LineDontBreak, 210},+ case lbAny | prBA<<32:+ return lbBA, LineDontBreak, 210+ case lbAny | prHY<<32:+ return lbHY, LineDontBreak, 210+ case lbAny | prNS<<32:+ return lbNS, LineDontBreak, 210+ case lbAny | prBB<<32:+ return lbBB, LineCanBreak, 310+ case lbBB | prAny<<32:+ return lbAny, LineDontBreak, 210 // LB21a.- {lbAny, prHL}: {lbHL, LineCanBreak, 310},- {lbHL, prHY}: {lbLB21a, LineDontBreak, 210},- {lbHL, prBA}: {lbLB21a, LineDontBreak, 210},- {lbLB21a, prAny}: {lbAny, LineDontBreak, 211},+ case lbAny | prHL<<32:+ return lbHL, LineCanBreak, 310+ case lbHL | prHY<<32:+ return lbLB21a, LineDontBreak, 210+ case lbHL | prBA<<32:+ return lbLB21a, LineDontBreak, 210+ case lbLB21a | prAny<<32:+ return lbAny, LineDontBreak, 211 // LB21b.- {lbSY, prHL}: {lbHL, LineDontBreak, 212},- {lbNUSY, prHL}: {lbHL, LineDontBreak, 212},+ case lbSY | prHL<<32:+ return lbHL, LineDontBreak, 212+ case lbNUSY | prHL<<32:+ return lbHL, LineDontBreak, 212 // LB22.- {lbAny, prIN}: {lbAny, LineDontBreak, 220},+ case lbAny | prIN<<32:+ return lbAny, LineDontBreak, 220 // LB23.- {lbAny, prAL}: {lbAL, LineCanBreak, 310},- {lbAny, prNU}: {lbNU, LineCanBreak, 310},- {lbAL, prNU}: {lbNU, LineDontBreak, 230},- {lbHL, prNU}: {lbNU, LineDontBreak, 230},- {lbNU, prAL}: {lbAL, LineDontBreak, 230},- {lbNU, prHL}: {lbHL, LineDontBreak, 230},- {lbNUNU, prAL}: {lbAL, LineDontBreak, 230},- {lbNUNU, prHL}: {lbHL, LineDontBreak, 230},+ case lbAny | prAL<<32:+ return lbAL, LineCanBreak, 310+ case lbAny | prNU<<32:+ return lbNU, LineCanBreak, 310+ case lbAL | prNU<<32:+ return lbNU, LineDontBreak, 230+ case lbHL | prNU<<32:+ return lbNU, LineDontBreak, 230+ case lbNU | prAL<<32:+ return lbAL, LineDontBreak, 230+ case lbNU | prHL<<32:+ return lbHL, LineDontBreak, 230+ case lbNUNU | prAL<<32:+ return lbAL, LineDontBreak, 230+ case lbNUNU | prHL<<32:+ return lbHL, LineDontBreak, 230 // LB23a.- {lbAny, prPR}: {lbPR, LineCanBreak, 310},- {lbAny, prID}: {lbIDEM, LineCanBreak, 310},- {lbAny, prEB}: {lbEB, LineCanBreak, 310},- {lbAny, prEM}: {lbIDEM, LineCanBreak, 310},- {lbPR, prID}: {lbIDEM, LineDontBreak, 231},- {lbPR, prEB}: {lbEB, LineDontBreak, 231},- {lbPR, prEM}: {lbIDEM, LineDontBreak, 231},- {lbIDEM, prPO}: {lbPO, LineDontBreak, 231},- {lbEB, prPO}: {lbPO, LineDontBreak, 231},+ case lbAny | prPR<<32:+ return lbPR, LineCanBreak, 310+ case lbAny | prID<<32:+ return lbIDEM, LineCanBreak, 310+ case lbAny | prEB<<32:+ return lbEB, LineCanBreak, 310+ case lbAny | prEM<<32:+ return lbIDEM, LineCanBreak, 310+ case lbPR | prID<<32:+ return lbIDEM, LineDontBreak, 231+ case lbPR | prEB<<32:+ return lbEB, LineDontBreak, 231+ case lbPR | prEM<<32:+ return lbIDEM, LineDontBreak, 231+ case lbIDEM | prPO<<32:+ return lbPO, LineDontBreak, 231+ case lbEB | prPO<<32:+ return lbPO, LineDontBreak, 231 // LB24.- {lbAny, prPO}: {lbPO, LineCanBreak, 310},- {lbPR, prAL}: {lbAL, LineDontBreak, 240},- {lbPR, prHL}: {lbHL, LineDontBreak, 240},- {lbPO, prAL}: {lbAL, LineDontBreak, 240},- {lbPO, prHL}: {lbHL, LineDontBreak, 240},- {lbAL, prPR}: {lbPR, LineDontBreak, 240},- {lbAL, prPO}: {lbPO, LineDontBreak, 240},- {lbHL, prPR}: {lbPR, LineDontBreak, 240},- {lbHL, prPO}: {lbPO, LineDontBreak, 240},+ case lbAny | prPO<<32:+ return lbPO, LineCanBreak, 310+ case lbPR | prAL<<32:+ return lbAL, LineDontBreak, 240+ case lbPR | prHL<<32:+ return lbHL, LineDontBreak, 240+ case lbPO | prAL<<32:+ return lbAL, LineDontBreak, 240+ case lbPO | prHL<<32:+ return lbHL, LineDontBreak, 240+ case lbAL | prPR<<32:+ return lbPR, LineDontBreak, 240+ case lbAL | prPO<<32:+ return lbPO, LineDontBreak, 240+ case lbHL | prPR<<32:+ return lbPR, LineDontBreak, 240+ case lbHL | prPO<<32:+ return lbPO, LineDontBreak, 240 // LB25 (simple transitions).- {lbPR, prNU}: {lbNU, LineDontBreak, 250},- {lbPO, prNU}: {lbNU, LineDontBreak, 250},- {lbOP, prNU}: {lbNU, LineDontBreak, 250},- {lbHY, prNU}: {lbNU, LineDontBreak, 250},- {lbNU, prNU}: {lbNUNU, LineDontBreak, 250},- {lbNU, prSY}: {lbNUSY, LineDontBreak, 250},- {lbNU, prIS}: {lbNUIS, LineDontBreak, 250},- {lbNUNU, prNU}: {lbNUNU, LineDontBreak, 250},- {lbNUNU, prSY}: {lbNUSY, LineDontBreak, 250},- {lbNUNU, prIS}: {lbNUIS, LineDontBreak, 250},- {lbNUSY, prNU}: {lbNUNU, LineDontBreak, 250},- {lbNUSY, prSY}: {lbNUSY, LineDontBreak, 250},- {lbNUSY, prIS}: {lbNUIS, LineDontBreak, 250},- {lbNUIS, prNU}: {lbNUNU, LineDontBreak, 250},- {lbNUIS, prSY}: {lbNUSY, LineDontBreak, 250},- {lbNUIS, prIS}: {lbNUIS, LineDontBreak, 250},- {lbNU, prCL}: {lbNUCL, LineDontBreak, 250},- {lbNU, prCP}: {lbNUCP, LineDontBreak, 250},- {lbNUNU, prCL}: {lbNUCL, LineDontBreak, 250},- {lbNUNU, prCP}: {lbNUCP, LineDontBreak, 250},- {lbNUSY, prCL}: {lbNUCL, LineDontBreak, 250},- {lbNUSY, prCP}: {lbNUCP, LineDontBreak, 250},- {lbNUIS, prCL}: {lbNUCL, LineDontBreak, 250},- {lbNUIS, prCP}: {lbNUCP, LineDontBreak, 250},- {lbNU, prPO}: {lbPO, LineDontBreak, 250},- {lbNUNU, prPO}: {lbPO, LineDontBreak, 250},- {lbNUSY, prPO}: {lbPO, LineDontBreak, 250},- {lbNUIS, prPO}: {lbPO, LineDontBreak, 250},- {lbNUCL, prPO}: {lbPO, LineDontBreak, 250},- {lbNUCP, prPO}: {lbPO, LineDontBreak, 250},- {lbNU, prPR}: {lbPR, LineDontBreak, 250},- {lbNUNU, prPR}: {lbPR, LineDontBreak, 250},- {lbNUSY, prPR}: {lbPR, LineDontBreak, 250},- {lbNUIS, prPR}: {lbPR, LineDontBreak, 250},- {lbNUCL, prPR}: {lbPR, LineDontBreak, 250},- {lbNUCP, prPR}: {lbPR, LineDontBreak, 250},+ case lbPR | prNU<<32:+ return lbNU, LineDontBreak, 250+ case lbPO | prNU<<32:+ return lbNU, LineDontBreak, 250+ case lbOP | prNU<<32:+ return lbNU, LineDontBreak, 250+ case lbHY | prNU<<32:+ return lbNU, LineDontBreak, 250+ case lbNU | prNU<<32:+ return lbNUNU, LineDontBreak, 250+ case lbNU | prSY<<32:+ return lbNUSY, LineDontBreak, 250+ case lbNU | prIS<<32:+ return lbNUIS, LineDontBreak, 250+ case lbNUNU | prNU<<32:+ return lbNUNU, LineDontBreak, 250+ case lbNUNU | prSY<<32:+ return lbNUSY, LineDontBreak, 250+ case lbNUNU | prIS<<32:+ return lbNUIS, LineDontBreak, 250+ case lbNUSY | prNU<<32:+ return lbNUNU, LineDontBreak, 250+ case lbNUSY | prSY<<32:+ return lbNUSY, LineDontBreak, 250+ case lbNUSY | prIS<<32:+ return lbNUIS, LineDontBreak, 250+ case lbNUIS | prNU<<32:+ return lbNUNU, LineDontBreak, 250+ case lbNUIS | prSY<<32:+ return lbNUSY, LineDontBreak, 250+ case lbNUIS | prIS<<32:+ return lbNUIS, LineDontBreak, 250+ case lbNU | prCL<<32:+ return lbNUCL, LineDontBreak, 250+ case lbNU | prCP<<32:+ return lbNUCP, LineDontBreak, 250+ case lbNUNU | prCL<<32:+ return lbNUCL, LineDontBreak, 250+ case lbNUNU | prCP<<32:+ return lbNUCP, LineDontBreak, 250+ case lbNUSY | prCL<<32:+ return lbNUCL, LineDontBreak, 250+ case lbNUSY | prCP<<32:+ return lbNUCP, LineDontBreak, 250+ case lbNUIS | prCL<<32:+ return lbNUCL, LineDontBreak, 250+ case lbNUIS | prCP<<32:+ return lbNUCP, LineDontBreak, 250+ case lbNU | prPO<<32:+ return lbPO, LineDontBreak, 250+ case lbNUNU | prPO<<32:+ return lbPO, LineDontBreak, 250+ case lbNUSY | prPO<<32:+ return lbPO, LineDontBreak, 250+ case lbNUIS | prPO<<32:+ return lbPO, LineDontBreak, 250+ case lbNUCL | prPO<<32:+ return lbPO, LineDontBreak, 250+ case lbNUCP | prPO<<32:+ return lbPO, LineDontBreak, 250+ case lbNU | prPR<<32:+ return lbPR, LineDontBreak, 250+ case lbNUNU | prPR<<32:+ return lbPR, LineDontBreak, 250+ case lbNUSY | prPR<<32:+ return lbPR, LineDontBreak, 250+ case lbNUIS | prPR<<32:+ return lbPR, LineDontBreak, 250+ case lbNUCL | prPR<<32:+ return lbPR, LineDontBreak, 250+ case lbNUCP | prPR<<32:+ return lbPR, LineDontBreak, 250 // LB26.- {lbAny, prJL}: {lbJL, LineCanBreak, 310},- {lbAny, prJV}: {lbJV, LineCanBreak, 310},- {lbAny, prJT}: {lbJT, LineCanBreak, 310},- {lbAny, prH2}: {lbH2, LineCanBreak, 310},- {lbAny, prH3}: {lbH3, LineCanBreak, 310},- {lbJL, prJL}: {lbJL, LineDontBreak, 260},- {lbJL, prJV}: {lbJV, LineDontBreak, 260},- {lbJL, prH2}: {lbH2, LineDontBreak, 260},- {lbJL, prH3}: {lbH3, LineDontBreak, 260},- {lbJV, prJV}: {lbJV, LineDontBreak, 260},- {lbJV, prJT}: {lbJT, LineDontBreak, 260},- {lbH2, prJV}: {lbJV, LineDontBreak, 260},- {lbH2, prJT}: {lbJT, LineDontBreak, 260},- {lbJT, prJT}: {lbJT, LineDontBreak, 260},- {lbH3, prJT}: {lbJT, LineDontBreak, 260},+ case lbAny | prJL<<32:+ return lbJL, LineCanBreak, 310+ case lbAny | prJV<<32:+ return lbJV, LineCanBreak, 310+ case lbAny | prJT<<32:+ return lbJT, LineCanBreak, 310+ case lbAny | prH2<<32:+ return lbH2, LineCanBreak, 310+ case lbAny | prH3<<32:+ return lbH3, LineCanBreak, 310+ case lbJL | prJL<<32:+ return lbJL, LineDontBreak, 260+ case lbJL | prJV<<32:+ return lbJV, LineDontBreak, 260+ case lbJL | prH2<<32:+ return lbH2, LineDontBreak, 260+ case lbJL | prH3<<32:+ return lbH3, LineDontBreak, 260+ case lbJV | prJV<<32:+ return lbJV, LineDontBreak, 260+ case lbJV | prJT<<32:+ return lbJT, LineDontBreak, 260+ case lbH2 | prJV<<32:+ return lbJV, LineDontBreak, 260+ case lbH2 | prJT<<32:+ return lbJT, LineDontBreak, 260+ case lbJT | prJT<<32:+ return lbJT, LineDontBreak, 260+ case lbH3 | prJT<<32:+ return lbJT, LineDontBreak, 260 // LB27.- {lbJL, prPO}: {lbPO, LineDontBreak, 270},- {lbJV, prPO}: {lbPO, LineDontBreak, 270},- {lbJT, prPO}: {lbPO, LineDontBreak, 270},- {lbH2, prPO}: {lbPO, LineDontBreak, 270},- {lbH3, prPO}: {lbPO, LineDontBreak, 270},- {lbPR, prJL}: {lbJL, LineDontBreak, 270},- {lbPR, prJV}: {lbJV, LineDontBreak, 270},- {lbPR, prJT}: {lbJT, LineDontBreak, 270},- {lbPR, prH2}: {lbH2, LineDontBreak, 270},- {lbPR, prH3}: {lbH3, LineDontBreak, 270},+ case lbJL | prPO<<32:+ return lbPO, LineDontBreak, 270+ case lbJV | prPO<<32:+ return lbPO, LineDontBreak, 270+ case lbJT | prPO<<32:+ return lbPO, LineDontBreak, 270+ case lbH2 | prPO<<32:+ return lbPO, LineDontBreak, 270+ case lbH3 | prPO<<32:+ return lbPO, LineDontBreak, 270+ case lbPR | prJL<<32:+ return lbJL, LineDontBreak, 270+ case lbPR | prJV<<32:+ return lbJV, LineDontBreak, 270+ case lbPR | prJT<<32:+ return lbJT, LineDontBreak, 270+ case lbPR | prH2<<32:+ return lbH2, LineDontBreak, 270+ case lbPR | prH3<<32:+ return lbH3, LineDontBreak, 270 // LB28.- {lbAL, prAL}: {lbAL, LineDontBreak, 280},- {lbAL, prHL}: {lbHL, LineDontBreak, 280},- {lbHL, prAL}: {lbAL, LineDontBreak, 280},- {lbHL, prHL}: {lbHL, LineDontBreak, 280},+ case lbAL | prAL<<32:+ return lbAL, LineDontBreak, 280+ case lbAL | prHL<<32:+ return lbHL, LineDontBreak, 280+ case lbHL | prAL<<32:+ return lbAL, LineDontBreak, 280+ case lbHL | prHL<<32:+ return lbHL, LineDontBreak, 280 // LB29.- {lbIS, prAL}: {lbAL, LineDontBreak, 290},- {lbIS, prHL}: {lbHL, LineDontBreak, 290},- {lbNUIS, prAL}: {lbAL, LineDontBreak, 290},- {lbNUIS, prHL}: {lbHL, LineDontBreak, 290},+ case lbIS | prAL<<32:+ return lbAL, LineDontBreak, 290+ case lbIS | prHL<<32:+ return lbHL, LineDontBreak, 290+ case lbNUIS | prAL<<32:+ return lbAL, LineDontBreak, 290+ case lbNUIS | prHL<<32:+ return lbHL, LineDontBreak, 290++ default:+ return -1, -1, -1+ } } // transitionLineBreakState determines the new state of the line break parser@@ -290,7 +449,7 @@ var lbTransitions = map[[2]int][3]int{ // further lookups. func transitionLineBreakState(state int, r rune, b []byte, str string) (newState int, lineBreak int) { // Determine the property of the next character.- nextProperty, generalCategory := propertyWithGenCat(lineBreakCodePoints, r)+ nextProperty, generalCategory := propertyLineBreak(r) // Prepare. var forceNoBreak, isCPeaFWH bool@@ -352,30 +511,27 @@ func transitionLineBreakState(state int, r rune, b []byte, str string) (newState // Find the applicable transition in the table. var rule int- transition, ok := lbTransitions[[2]int{state, nextProperty}]- if ok {- // We have a specific transition. We'll use it.- newState, lineBreak, rule = transition[0], transition[1], transition[2]- } else {+ newState, lineBreak, rule = lbTransitions(state, nextProperty)+ if newState < 0 { // No specific transition found. Try the less specific ones.- transAnyProp, okAnyProp := lbTransitions[[2]int{state, prAny}]- transAnyState, okAnyState := lbTransitions[[2]int{lbAny, nextProperty}]- if okAnyProp && okAnyState {+ anyPropProp, anyPropLineBreak, anyPropRule := lbTransitions(state, prAny)+ anyStateProp, anyStateLineBreak, anyStateRule := lbTransitions(lbAny, nextProperty)+ if anyPropProp >= 0 && anyStateProp >= 0 { // Both apply. We'll use a mix (see comments for grTransitions).- newState, lineBreak, rule = transAnyState[0], transAnyState[1], transAnyState[2]- if transAnyProp[2] < transAnyState[2] {- lineBreak, rule = transAnyProp[1], transAnyProp[2]+ newState, lineBreak, rule = anyStateProp, anyStateLineBreak, anyStateRule+ if anyPropRule < anyStateRule {+ lineBreak, rule = anyPropLineBreak, anyPropRule }- } else if okAnyProp {+ } else if anyPropProp >= 0 { // We only have a specific state.- newState, lineBreak, rule = transAnyProp[0], transAnyProp[1], transAnyProp[2]+ newState, lineBreak, rule = anyPropProp, anyPropLineBreak, anyPropRule // This branch will probably never be reached because okAnyState will // always be true given the current transition map. But we keep it here // for future modifications to the transition map where this may not be // true anymore.- } else if okAnyState {+ } else if anyStateProp >= 0 { // We only have a specific property.- newState, lineBreak, rule = transAnyState[0], transAnyState[1], transAnyState[2]+ newState, lineBreak, rule = anyStateProp, anyStateLineBreak, anyStateRule } else { // No known transition. LB31: ALL ÷ ALL. newState, lineBreak, rule = lbAny, LineCanBreak, 310@@ -414,7 +570,7 @@ func transitionLineBreakState(state int, r rune, b []byte, str string) (newState r, _ = utf8.DecodeRuneInString(str) } if r != utf8.RuneError {- pr, _ := propertyWithGenCat(lineBreakCodePoints, r)+ pr, _ := propertyLineBreak(r) if pr == prNU { return lbNU, LineDontBreak }
@@ -160,10 +160,20 @@ func property(dictionary [][3]int, r rune) int { return propertySearch(dictionary, r)[2] } -// propertyWithGenCat returns the Unicode property value and General Category-// (see constants above) of the given code point.-func propertyWithGenCat(dictionary [][4]int, r rune) (property, generalCategory int) {- entry := propertySearch(dictionary, r)+// propertyLineBreak returns the Unicode property value and General Category+// (see constants above) of the given code point, as listed in the line break+// code points table, while fast tracking ASCII digits and letters.+func propertyLineBreak(r rune) (property, generalCategory int) {+ if r >= 'a' && r <= 'z' {+ return prAL, gcLl+ }+ if r >= 'A' && r <= 'Z' {+ return prAL, gcLu+ }+ if r >= '0' && r <= '9' {+ return prNU, gcNd+ }+ entry := propertySearch(lineBreakCodePoints, r) return entry[2], entry[3] }
@@ -18,104 +18,178 @@ const ( sbSB8aSp ) -// The sentence break parser's breaking instructions.-const (- sbDontBreak = iota- sbBreak-)--// The sentence break parser's state transitions. It's anologous to-// grTransitions, see comments there for details. Unicode version 14.0.0.-var sbTransitions = map[[2]int][3]int{+// sbTransitions implements the sentence break parser's state transitions. It's+// anologous to [grTransitions], see comments there for details.+//+// Unicode version 15.0.0.+func sbTransitions(state, prop int) (newState int, sentenceBreak bool, rule int) {+ switch uint64(state) | uint64(prop)<<32 { // SB3.- {sbAny, prCR}: {sbCR, sbDontBreak, 9990},- {sbCR, prLF}: {sbParaSep, sbDontBreak, 30},+ case sbAny | prCR<<32:+ return sbCR, false, 9990+ case sbCR | prLF<<32:+ return sbParaSep, false, 30 // SB4.- {sbAny, prSep}: {sbParaSep, sbDontBreak, 9990},- {sbAny, prLF}: {sbParaSep, sbDontBreak, 9990},- {sbParaSep, prAny}: {sbAny, sbBreak, 40},- {sbCR, prAny}: {sbAny, sbBreak, 40},+ case sbAny | prSep<<32:+ return sbParaSep, false, 9990+ case sbAny | prLF<<32:+ return sbParaSep, false, 9990+ case sbParaSep | prAny<<32:+ return sbAny, true, 40+ case sbCR | prAny<<32:+ return sbAny, true, 40 // SB6.- {sbAny, prATerm}: {sbATerm, sbDontBreak, 9990},- {sbATerm, prNumeric}: {sbAny, sbDontBreak, 60},- {sbSB7, prNumeric}: {sbAny, sbDontBreak, 60}, // Because ATerm also appears in SB7.+ case sbAny | prATerm<<32:+ return sbATerm, false, 9990+ case sbATerm | prNumeric<<32:+ return sbAny, false, 60+ case sbSB7 | prNumeric<<32:+ return sbAny, false, 60 // Because ATerm also appears in SB7. // SB7.- {sbAny, prUpper}: {sbUpper, sbDontBreak, 9990},- {sbAny, prLower}: {sbLower, sbDontBreak, 9990},- {sbUpper, prATerm}: {sbSB7, sbDontBreak, 70},- {sbLower, prATerm}: {sbSB7, sbDontBreak, 70},- {sbSB7, prUpper}: {sbUpper, sbDontBreak, 70},+ case sbAny | prUpper<<32:+ return sbUpper, false, 9990+ case sbAny | prLower<<32:+ return sbLower, false, 9990+ case sbUpper | prATerm<<32:+ return sbSB7, false, 70+ case sbLower | prATerm<<32:+ return sbSB7, false, 70+ case sbSB7 | prUpper<<32:+ return sbUpper, false, 70 // SB8a.- {sbAny, prSTerm}: {sbSTerm, sbDontBreak, 9990},- {sbATerm, prSContinue}: {sbAny, sbDontBreak, 81},- {sbATerm, prATerm}: {sbATerm, sbDontBreak, 81},- {sbATerm, prSTerm}: {sbSTerm, sbDontBreak, 81},- {sbSB7, prSContinue}: {sbAny, sbDontBreak, 81},- {sbSB7, prATerm}: {sbATerm, sbDontBreak, 81},- {sbSB7, prSTerm}: {sbSTerm, sbDontBreak, 81},- {sbSB8Close, prSContinue}: {sbAny, sbDontBreak, 81},- {sbSB8Close, prATerm}: {sbATerm, sbDontBreak, 81},- {sbSB8Close, prSTerm}: {sbSTerm, sbDontBreak, 81},- {sbSB8Sp, prSContinue}: {sbAny, sbDontBreak, 81},- {sbSB8Sp, prATerm}: {sbATerm, sbDontBreak, 81},- {sbSB8Sp, prSTerm}: {sbSTerm, sbDontBreak, 81},- {sbSTerm, prSContinue}: {sbAny, sbDontBreak, 81},- {sbSTerm, prATerm}: {sbATerm, sbDontBreak, 81},- {sbSTerm, prSTerm}: {sbSTerm, sbDontBreak, 81},- {sbSB8aClose, prSContinue}: {sbAny, sbDontBreak, 81},- {sbSB8aClose, prATerm}: {sbATerm, sbDontBreak, 81},- {sbSB8aClose, prSTerm}: {sbSTerm, sbDontBreak, 81},- {sbSB8aSp, prSContinue}: {sbAny, sbDontBreak, 81},- {sbSB8aSp, prATerm}: {sbATerm, sbDontBreak, 81},- {sbSB8aSp, prSTerm}: {sbSTerm, sbDontBreak, 81},+ case sbAny | prSTerm<<32:+ return sbSTerm, false, 9990+ case sbATerm | prSContinue<<32:+ return sbAny, false, 81+ case sbATerm | prATerm<<32:+ return sbATerm, false, 81+ case sbATerm | prSTerm<<32:+ return sbSTerm, false, 81+ case sbSB7 | prSContinue<<32:+ return sbAny, false, 81+ case sbSB7 | prATerm<<32:+ return sbATerm, false, 81+ case sbSB7 | prSTerm<<32:+ return sbSTerm, false, 81+ case sbSB8Close | prSContinue<<32:+ return sbAny, false, 81+ case sbSB8Close | prATerm<<32:+ return sbATerm, false, 81+ case sbSB8Close | prSTerm<<32:+ return sbSTerm, false, 81+ case sbSB8Sp | prSContinue<<32:+ return sbAny, false, 81+ case sbSB8Sp | prATerm<<32:+ return sbATerm, false, 81+ case sbSB8Sp | prSTerm<<32:+ return sbSTerm, false, 81+ case sbSTerm | prSContinue<<32:+ return sbAny, false, 81+ case sbSTerm | prATerm<<32:+ return sbATerm, false, 81+ case sbSTerm | prSTerm<<32:+ return sbSTerm, false, 81+ case sbSB8aClose | prSContinue<<32:+ return sbAny, false, 81+ case sbSB8aClose | prATerm<<32:+ return sbATerm, false, 81+ case sbSB8aClose | prSTerm<<32:+ return sbSTerm, false, 81+ case sbSB8aSp | prSContinue<<32:+ return sbAny, false, 81+ case sbSB8aSp | prATerm<<32:+ return sbATerm, false, 81+ case sbSB8aSp | prSTerm<<32:+ return sbSTerm, false, 81 // SB9.- {sbATerm, prClose}: {sbSB8Close, sbDontBreak, 90},- {sbSB7, prClose}: {sbSB8Close, sbDontBreak, 90},- {sbSB8Close, prClose}: {sbSB8Close, sbDontBreak, 90},- {sbATerm, prSp}: {sbSB8Sp, sbDontBreak, 90},- {sbSB7, prSp}: {sbSB8Sp, sbDontBreak, 90},- {sbSB8Close, prSp}: {sbSB8Sp, sbDontBreak, 90},- {sbSTerm, prClose}: {sbSB8aClose, sbDontBreak, 90},- {sbSB8aClose, prClose}: {sbSB8aClose, sbDontBreak, 90},- {sbSTerm, prSp}: {sbSB8aSp, sbDontBreak, 90},- {sbSB8aClose, prSp}: {sbSB8aSp, sbDontBreak, 90},- {sbATerm, prSep}: {sbParaSep, sbDontBreak, 90},- {sbATerm, prCR}: {sbParaSep, sbDontBreak, 90},- {sbATerm, prLF}: {sbParaSep, sbDontBreak, 90},- {sbSB7, prSep}: {sbParaSep, sbDontBreak, 90},- {sbSB7, prCR}: {sbParaSep, sbDontBreak, 90},- {sbSB7, prLF}: {sbParaSep, sbDontBreak, 90},- {sbSB8Close, prSep}: {sbParaSep, sbDontBreak, 90},- {sbSB8Close, prCR}: {sbParaSep, sbDontBreak, 90},- {sbSB8Close, prLF}: {sbParaSep, sbDontBreak, 90},- {sbSTerm, prSep}: {sbParaSep, sbDontBreak, 90},- {sbSTerm, prCR}: {sbParaSep, sbDontBreak, 90},- {sbSTerm, prLF}: {sbParaSep, sbDontBreak, 90},- {sbSB8aClose, prSep}: {sbParaSep, sbDontBreak, 90},- {sbSB8aClose, prCR}: {sbParaSep, sbDontBreak, 90},- {sbSB8aClose, prLF}: {sbParaSep, sbDontBreak, 90},+ case sbATerm | prClose<<32:+ return sbSB8Close, false, 90+ case sbSB7 | prClose<<32:+ return sbSB8Close, false, 90+ case sbSB8Close | prClose<<32:+ return sbSB8Close, false, 90+ case sbATerm | prSp<<32:+ return sbSB8Sp, false, 90+ case sbSB7 | prSp<<32:+ return sbSB8Sp, false, 90+ case sbSB8Close | prSp<<32:+ return sbSB8Sp, false, 90+ case sbSTerm | prClose<<32:+ return sbSB8aClose, false, 90+ case sbSB8aClose | prClose<<32:+ return sbSB8aClose, false, 90+ case sbSTerm | prSp<<32:+ return sbSB8aSp, false, 90+ case sbSB8aClose | prSp<<32:+ return sbSB8aSp, false, 90+ case sbATerm | prSep<<32:+ return sbParaSep, false, 90+ case sbATerm | prCR<<32:+ return sbParaSep, false, 90+ case sbATerm | prLF<<32:+ return sbParaSep, false, 90+ case sbSB7 | prSep<<32:+ return sbParaSep, false, 90+ case sbSB7 | prCR<<32:+ return sbParaSep, false, 90+ case sbSB7 | prLF<<32:+ return sbParaSep, false, 90+ case sbSB8Close | prSep<<32:+ return sbParaSep, false, 90+ case sbSB8Close | prCR<<32:+ return sbParaSep, false, 90+ case sbSB8Close | prLF<<32:+ return sbParaSep, false, 90+ case sbSTerm | prSep<<32:+ return sbParaSep, false, 90+ case sbSTerm | prCR<<32:+ return sbParaSep, false, 90+ case sbSTerm | prLF<<32:+ return sbParaSep, false, 90+ case sbSB8aClose | prSep<<32:+ return sbParaSep, false, 90+ case sbSB8aClose | prCR<<32:+ return sbParaSep, false, 90+ case sbSB8aClose | prLF<<32:+ return sbParaSep, false, 90 // SB10.- {sbSB8Sp, prSp}: {sbSB8Sp, sbDontBreak, 100},- {sbSB8aSp, prSp}: {sbSB8aSp, sbDontBreak, 100},- {sbSB8Sp, prSep}: {sbParaSep, sbDontBreak, 100},- {sbSB8Sp, prCR}: {sbParaSep, sbDontBreak, 100},- {sbSB8Sp, prLF}: {sbParaSep, sbDontBreak, 100},+ case sbSB8Sp | prSp<<32:+ return sbSB8Sp, false, 100+ case sbSB8aSp | prSp<<32:+ return sbSB8aSp, false, 100+ case sbSB8Sp | prSep<<32:+ return sbParaSep, false, 100+ case sbSB8Sp | prCR<<32:+ return sbParaSep, false, 100+ case sbSB8Sp | prLF<<32:+ return sbParaSep, false, 100 // SB11.- {sbATerm, prAny}: {sbAny, sbBreak, 110},- {sbSB7, prAny}: {sbAny, sbBreak, 110},- {sbSB8Close, prAny}: {sbAny, sbBreak, 110},- {sbSB8Sp, prAny}: {sbAny, sbBreak, 110},- {sbSTerm, prAny}: {sbAny, sbBreak, 110},- {sbSB8aClose, prAny}: {sbAny, sbBreak, 110},- {sbSB8aSp, prAny}: {sbAny, sbBreak, 110},+ case sbATerm | prAny<<32:+ return sbAny, true, 110+ case sbSB7 | prAny<<32:+ return sbAny, true, 110+ case sbSB8Close | prAny<<32:+ return sbAny, true, 110+ case sbSB8Sp | prAny<<32:+ return sbAny, true, 110+ case sbSTerm | prAny<<32:+ return sbAny, true, 110+ case sbSB8aClose | prAny<<32:+ return sbAny, true, 110+ case sbSB8aSp | prAny<<32:+ return sbAny, true, 110 // We'll always break after ParaSep due to SB4.++ default:+ return -1, false, -1+ } } // transitionSentenceBreakState determines the new state of the sentence break@@ -141,30 +215,27 @@ func transitionSentenceBreakState(state int, r rune, b []byte, str string) (newS // Find the applicable transition in the table. var rule int- transition, ok := sbTransitions[[2]int{state, nextProperty}]- if ok {- // We have a specific transition. We'll use it.- newState, sentenceBreak, rule = transition[0], transition[1] == sbBreak, transition[2]- } else {+ newState, sentenceBreak, rule = sbTransitions(state, nextProperty)+ if newState < 0 { // No specific transition found. Try the less specific ones.- transAnyProp, okAnyProp := sbTransitions[[2]int{state, prAny}]- transAnyState, okAnyState := sbTransitions[[2]int{sbAny, nextProperty}]- if okAnyProp && okAnyState {+ anyPropState, anyPropProp, anyPropRule := sbTransitions(state, prAny)+ anyStateState, anyStateProp, anyStateRule := sbTransitions(sbAny, nextProperty)+ if anyPropState >= 0 && anyStateState >= 0 { // Both apply. We'll use a mix (see comments for grTransitions).- newState, sentenceBreak, rule = transAnyState[0], transAnyState[1] == sbBreak, transAnyState[2]- if transAnyProp[2] < transAnyState[2] {- sentenceBreak, rule = transAnyProp[1] == sbBreak, transAnyProp[2]+ newState, sentenceBreak, rule = anyStateState, anyStateProp, anyStateRule+ if anyPropRule < anyStateRule {+ sentenceBreak, rule = anyPropProp, anyPropRule }- } else if okAnyProp {+ } else if anyPropState >= 0 { // We only have a specific state.- newState, sentenceBreak, rule = transAnyProp[0], transAnyProp[1] == sbBreak, transAnyProp[2]+ newState, sentenceBreak, rule = anyPropState, anyPropProp, anyPropRule // This branch will probably never be reached because okAnyState will // always be true given the current transition map. But we keep it here // for future modifications to the transition map where this may not be // true anymore.- } else if okAnyState {+ } else if anyStateState >= 0 { // We only have a specific property.- newState, sentenceBreak, rule = transAnyState[0], transAnyState[1] == sbBreak, transAnyState[2]+ newState, sentenceBreak, rule = anyStateState, anyStateProp, anyStateRule } else { // No known transition. SB999: Any × Any. newState, sentenceBreak, rule = sbAny, false, 9990
@@ -22,82 +22,121 @@ const ( wbZWJBit = 16 // This bit is set for any states followed by at least one zero-width joiner (see WB4 and WB3c). ) -// The word break parser's breaking instructions.-const (- wbDontBreak = iota- wbBreak-)--// The word break parser's state transitions. It's anologous to grTransitions,-// see comments there for details. Unicode version 15.0.0.-var wbTransitions = map[[2]int][3]int{+// wbTransitions implements the word break parser's state transitions. It's+// anologous to [grTransitions], see comments there for details.+//+// Unicode version 15.0.0.+func wbTransitions(state, prop int) (newState int, wordBreak bool, rule int) {+ switch uint64(state) | uint64(prop)<<32 { // WB3b.- {wbAny, prNewline}: {wbNewline, wbBreak, 32},- {wbAny, prCR}: {wbCR, wbBreak, 32},- {wbAny, prLF}: {wbLF, wbBreak, 32},+ case wbAny | prNewline<<32:+ return wbNewline, true, 32+ case wbAny | prCR<<32:+ return wbCR, true, 32+ case wbAny | prLF<<32:+ return wbLF, true, 32 // WB3a.- {wbNewline, prAny}: {wbAny, wbBreak, 31},- {wbCR, prAny}: {wbAny, wbBreak, 31},- {wbLF, prAny}: {wbAny, wbBreak, 31},+ case wbNewline | prAny<<32:+ return wbAny, true, 31+ case wbCR | prAny<<32:+ return wbAny, true, 31+ case wbLF | prAny<<32:+ return wbAny, true, 31 // WB3.- {wbCR, prLF}: {wbLF, wbDontBreak, 30},+ case wbCR | prLF<<32:+ return wbLF, false, 30 // WB3d.- {wbAny, prWSegSpace}: {wbWSegSpace, wbBreak, 9990},- {wbWSegSpace, prWSegSpace}: {wbWSegSpace, wbDontBreak, 34},+ case wbAny | prWSegSpace<<32:+ return wbWSegSpace, true, 9990+ case wbWSegSpace | prWSegSpace<<32:+ return wbWSegSpace, false, 34 // WB5.- {wbAny, prALetter}: {wbALetter, wbBreak, 9990},- {wbAny, prHebrewLetter}: {wbHebrewLetter, wbBreak, 9990},- {wbALetter, prALetter}: {wbALetter, wbDontBreak, 50},- {wbALetter, prHebrewLetter}: {wbHebrewLetter, wbDontBreak, 50},- {wbHebrewLetter, prALetter}: {wbALetter, wbDontBreak, 50},- {wbHebrewLetter, prHebrewLetter}: {wbHebrewLetter, wbDontBreak, 50},+ case wbAny | prALetter<<32:+ return wbALetter, true, 9990+ case wbAny | prHebrewLetter<<32:+ return wbHebrewLetter, true, 9990+ case wbALetter | prALetter<<32:+ return wbALetter, false, 50+ case wbALetter | prHebrewLetter<<32:+ return wbHebrewLetter, false, 50+ case wbHebrewLetter | prALetter<<32:+ return wbALetter, false, 50+ case wbHebrewLetter | prHebrewLetter<<32:+ return wbHebrewLetter, false, 50 // WB7. Transitions to wbWB7 handled by transitionWordBreakState().- {wbWB7, prALetter}: {wbALetter, wbDontBreak, 70},- {wbWB7, prHebrewLetter}: {wbHebrewLetter, wbDontBreak, 70},+ case wbWB7 | prALetter<<32:+ return wbALetter, false, 70+ case wbWB7 | prHebrewLetter<<32:+ return wbHebrewLetter, false, 70 // WB7a.- {wbHebrewLetter, prSingleQuote}: {wbAny, wbDontBreak, 71},+ case wbHebrewLetter | prSingleQuote<<32:+ return wbAny, false, 71 // WB7c. Transitions to wbWB7c handled by transitionWordBreakState().- {wbWB7c, prHebrewLetter}: {wbHebrewLetter, wbDontBreak, 73},+ case wbWB7c | prHebrewLetter<<32:+ return wbHebrewLetter, false, 73 // WB8.- {wbAny, prNumeric}: {wbNumeric, wbBreak, 9990},- {wbNumeric, prNumeric}: {wbNumeric, wbDontBreak, 80},+ case wbAny | prNumeric<<32:+ return wbNumeric, true, 9990+ case wbNumeric | prNumeric<<32:+ return wbNumeric, false, 80 // WB9.- {wbALetter, prNumeric}: {wbNumeric, wbDontBreak, 90},- {wbHebrewLetter, prNumeric}: {wbNumeric, wbDontBreak, 90},+ case wbALetter | prNumeric<<32:+ return wbNumeric, false, 90+ case wbHebrewLetter | prNumeric<<32:+ return wbNumeric, false, 90 // WB10.- {wbNumeric, prALetter}: {wbALetter, wbDontBreak, 100},- {wbNumeric, prHebrewLetter}: {wbHebrewLetter, wbDontBreak, 100},+ case wbNumeric | prALetter<<32:+ return wbALetter, false, 100+ case wbNumeric | prHebrewLetter<<32:+ return wbHebrewLetter, false, 100 // WB11. Transitions to wbWB11 handled by transitionWordBreakState().- {wbWB11, prNumeric}: {wbNumeric, wbDontBreak, 110},+ case wbWB11 | prNumeric<<32:+ return wbNumeric, false, 110 // WB13.- {wbAny, prKatakana}: {wbKatakana, wbBreak, 9990},- {wbKatakana, prKatakana}: {wbKatakana, wbDontBreak, 130},+ case wbAny | prKatakana<<32:+ return wbKatakana, true, 9990+ case wbKatakana | prKatakana<<32:+ return wbKatakana, false, 130 // WB13a.- {wbAny, prExtendNumLet}: {wbExtendNumLet, wbBreak, 9990},- {wbALetter, prExtendNumLet}: {wbExtendNumLet, wbDontBreak, 131},- {wbHebrewLetter, prExtendNumLet}: {wbExtendNumLet, wbDontBreak, 131},- {wbNumeric, prExtendNumLet}: {wbExtendNumLet, wbDontBreak, 131},- {wbKatakana, prExtendNumLet}: {wbExtendNumLet, wbDontBreak, 131},- {wbExtendNumLet, prExtendNumLet}: {wbExtendNumLet, wbDontBreak, 131},+ case wbAny | prExtendNumLet<<32:+ return wbExtendNumLet, true, 9990+ case wbALetter | prExtendNumLet<<32:+ return wbExtendNumLet, false, 131+ case wbHebrewLetter | prExtendNumLet<<32:+ return wbExtendNumLet, false, 131+ case wbNumeric | prExtendNumLet<<32:+ return wbExtendNumLet, false, 131+ case wbKatakana | prExtendNumLet<<32:+ return wbExtendNumLet, false, 131+ case wbExtendNumLet | prExtendNumLet<<32:+ return wbExtendNumLet, false, 131 // WB13b.- {wbExtendNumLet, prALetter}: {wbALetter, wbDontBreak, 132},- {wbExtendNumLet, prHebrewLetter}: {wbHebrewLetter, wbDontBreak, 132},- {wbExtendNumLet, prNumeric}: {wbNumeric, wbDontBreak, 132},- {wbExtendNumLet, prKatakana}: {wbKatakana, wbDontBreak, 132},+ case wbExtendNumLet | prALetter<<32:+ return wbALetter, false, 132+ case wbExtendNumLet | prHebrewLetter<<32:+ return wbHebrewLetter, false, 132+ case wbExtendNumLet | prNumeric<<32:+ return wbNumeric, false, 132+ case wbExtendNumLet | prKatakana<<32:+ return wbKatakana, false, 132++ default:+ return -1, false, -1+ } } // transitionWordBreakState determines the new state of the word break parser@@ -141,30 +180,27 @@ func transitionWordBreakState(state int, r rune, b []byte, str string) (newState // Find the applicable transition in the table. var rule int- transition, ok := wbTransitions[[2]int{state, nextProperty}]- if ok {- // We have a specific transition. We'll use it.- newState, wordBreak, rule = transition[0], transition[1] == wbBreak, transition[2]- } else {+ newState, wordBreak, rule = wbTransitions(state, nextProperty)+ if newState < 0 { // No specific transition found. Try the less specific ones.- transAnyProp, okAnyProp := wbTransitions[[2]int{state, prAny}]- transAnyState, okAnyState := wbTransitions[[2]int{wbAny, nextProperty}]- if okAnyProp && okAnyState {+ anyPropState, anyPropWordBreak, anyPropRule := wbTransitions(state, prAny)+ anyStateState, anyStateWordBreak, anyStateRule := wbTransitions(wbAny, nextProperty)+ if anyPropState >= 0 && anyStateState >= 0 { // Both apply. We'll use a mix (see comments for grTransitions).- newState, wordBreak, rule = transAnyState[0], transAnyState[1] == wbBreak, transAnyState[2]- if transAnyProp[2] < transAnyState[2] {- wordBreak, rule = transAnyProp[1] == wbBreak, transAnyProp[2]+ newState, wordBreak, rule = anyStateState, anyStateWordBreak, anyStateRule+ if anyPropRule < anyStateRule {+ wordBreak, rule = anyPropWordBreak, anyPropRule }- } else if okAnyProp {+ } else if anyPropState >= 0 { // We only have a specific state.- newState, wordBreak, rule = transAnyProp[0], transAnyProp[1] == wbBreak, transAnyProp[2]+ newState, wordBreak, rule = anyPropState, anyPropWordBreak, anyPropRule // This branch will probably never be reached because okAnyState will // always be true given the current transition map. But we keep it here // for future modifications to the transition map where this may not be // true anymore.- } else if okAnyState {+ } else if anyStateState >= 0 { // We only have a specific property.- newState, wordBreak, rule = transAnyState[0], transAnyState[1] == wbBreak, transAnyState[2]+ newState, wordBreak, rule = anyStateState, anyStateWordBreak, anyStateRule } else { // No known transition. WB999: Any ÷ Any. newState, wordBreak, rule = wbAny, true, 9990