OSDN Git Service

libgo: update to weekly.2011-10-25
authorian <ian@138bc75d-0d04-0410-961f-82ee72b054a4>
Tue, 29 Nov 2011 23:02:54 +0000 (23:02 +0000)
committerian <ian@138bc75d-0d04-0410-961f-82ee72b054a4>
Tue, 29 Nov 2011 23:02:54 +0000 (23:02 +0000)
Changes were mainly straightforward to merge.

git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@181824 138bc75d-0d04-0410-961f-82ee72b054a4

39 files changed:
libgo/MERGE
libgo/Makefile.am
libgo/Makefile.in
libgo/go/big/int.go
libgo/go/big/int_test.go
libgo/go/big/nat.go
libgo/go/big/nat_test.go
libgo/go/big/rat.go
libgo/go/big/rat_test.go
libgo/go/crypto/x509/x509.go
libgo/go/crypto/x509/x509_test.go
libgo/go/exp/inotify/inotify_linux.go [moved from libgo/go/os/inotify/inotify_linux.go with 100% similarity]
libgo/go/exp/inotify/inotify_linux_test.go [moved from libgo/go/os/inotify/inotify_linux_test.go with 100% similarity]
libgo/go/exp/ssh/channel.go
libgo/go/exp/ssh/client.go [new file with mode: 0644]
libgo/go/exp/ssh/doc.go
libgo/go/exp/ssh/messages.go
libgo/go/exp/ssh/server.go
libgo/go/exp/ssh/session.go [new file with mode: 0644]
libgo/go/exp/ssh/transport.go
libgo/go/exp/ssh/transport_test.go
libgo/go/exp/types/gcimporter.go
libgo/go/exp/winfsnotify/winfsnotify_test.go
libgo/go/fmt/fmt_test.go
libgo/go/fmt/print.go
libgo/go/go/ast/print_test.go
libgo/go/html/doc.go
libgo/go/html/parse.go
libgo/go/html/parse_test.go
libgo/go/html/render.go
libgo/go/html/token.go
libgo/go/html/token_test.go
libgo/go/http/client.go
libgo/go/http/client_test.go
libgo/go/http/doc.go [new file with mode: 0644]
libgo/go/http/request.go
libgo/go/http/transport.go
libgo/go/net/sock_windows.go
libgo/go/unicode/tables.go

index 8ba17de..2a86f2f 100644 (file)
@@ -1,4 +1,4 @@
-6d7136d74b65
+941b8015061a
 
 The first line of this file holds the Mercurial revision number of the
 last merge done from the master library sources.
index c30d9c3..5ab10a6 100644 (file)
@@ -236,11 +236,19 @@ toolexeclibgoencoding_DATA = \
        encoding/hex.gox \
        encoding/pem.gox
 
+if LIBGO_IS_LINUX
+# exp_inotify_gox = exp/inotify.gox
+exp_inotify_gox =
+else
+exp_inotify_gox =
+endif
+
 toolexeclibgoexpdir = $(toolexeclibgodir)/exp
 
 toolexeclibgoexp_DATA = \
        exp/ebnf.gox \
        exp/gui.gox \
+       $(exp_inotify_gox) \
        exp/norm.gox \
        exp/spdy.gox \
        exp/sql.gox \
@@ -332,15 +340,7 @@ toolexeclibgoold_DATA = \
 
 toolexeclibgoosdir = $(toolexeclibgodir)/os
 
-if LIBGO_IS_LINUX
-# os_inotify_gox = os/inotify.gox
-os_inotify_gox =
-else
-os_inotify_gox =
-endif
-
 toolexeclibgoos_DATA = \
-       $(os_inotify_gox) \
        os/user.gox \
        os/signal.gox
 
@@ -1212,6 +1212,8 @@ go_exp_ebnf_files = \
        go/exp/ebnf/parser.go
 go_exp_gui_files = \
        go/exp/gui/gui.go
+go_exp_inotify_files = \
+       go/exp/inotify/inotify_linux.go
 go_exp_norm_files = \
        go/exp/norm/composition.go \
        go/exp/norm/forminfo.go \
@@ -1229,11 +1231,13 @@ go_exp_sql_files = \
        go/exp/sql/sql.go
 go_exp_ssh_files = \
        go/exp/ssh/channel.go \
+       go/exp/ssh/client.go \
        go/exp/ssh/common.go \
        go/exp/ssh/doc.go \
        go/exp/ssh/messages.go \
        go/exp/ssh/server.go \
        go/exp/ssh/server_shell.go \
+       go/exp/ssh/session.go \
        go/exp/ssh/transport.go
 go_exp_terminal_files = \
        go/exp/terminal/shell.go \
@@ -1387,9 +1391,6 @@ go_old_template_files = \
        go/old/template/format.go \
        go/old/template/parse.go
 
-go_os_inotify_files = \
-       go/os/inotify/inotify_linux.go
-
 go_os_user_files = \
        go/os/user/user.go \
        go/os/user/lookup_unix.go
@@ -2723,6 +2724,13 @@ exp/gui/x11/check: $(CHECK_DEPS)
        @$(CHECK)
 .PHONY: exp/gui/x11/check
 
+exp/inotify.lo: $(go_exp_inotify_files) fmt.gox os.gox strings.gox syscall.gox
+       $(BUILDPACKAGE)
+exp/inotify/check: $(CHECK_DEPS)
+       @$(MKDIR_P) exp/inotify
+       @$(CHECK)
+.PHONY: exp/inotify/check
+
 exp/sql/driver.lo: $(go_exp_sql_driver_files) fmt.gox os.gox reflect.gox \
                strconv.gox
        $(BUILDPACKAGE)
@@ -2998,13 +3006,6 @@ old/template/check: $(CHECK_DEPS)
        @$(CHECK)
 .PHONY: old/template/check
 
-os/inotify.lo: $(go_os_inotify_files) fmt.gox os.gox strings.gox syscall.gox
-       $(BUILDPACKAGE)
-os/inotify/check: $(CHECK_DEPS)
-       @$(MKDIR_P) os/inotify
-       @$(CHECK)
-.PHONY: os/inotify/check
-
 os/user.lo: $(go_os_user_files) fmt.gox os.gox runtime.gox strconv.gox \
                strings.gox syscall.gox
        $(BUILDPACKAGE)
@@ -3331,6 +3332,8 @@ exp/ebnf.gox: exp/ebnf.lo
        $(BUILDGOX)
 exp/gui.gox: exp/gui.lo
        $(BUILDGOX)
+exp/inotify.gox: exp/inotify.lo
+       $(BUILDGOX)
 exp/norm.gox: exp/norm.lo
        $(BUILDGOX)
 exp/spdy.gox: exp/spdy.lo
@@ -3424,8 +3427,6 @@ old/regexp.gox: old/regexp.lo
 old/template.gox: old/template.lo
        $(BUILDGOX)
 
-os/inotify.gox: os/inotify.lo
-       $(BUILDGOX)
 os/user.gox: os/user.lo
        $(BUILDGOX)
 os/signal.gox: os/signal.lo
@@ -3459,10 +3460,10 @@ testing/script.gox: testing/script.lo
        $(BUILDGOX)
 
 if LIBGO_IS_LINUX
-# os_inotify_check = os/inotify/check
-os_inotify_check =
+# exp_inotify_check = exp/inotify/check
+exp_inotify_check =
 else
-os_inotify_check =
+exp_inotify_check =
 endif
 
 TEST_PACKAGES = \
@@ -3563,6 +3564,7 @@ TEST_PACKAGES = \
        encoding/hex/check \
        encoding/pem/check \
        exp/ebnf/check \
+       $(exp_inotify_check) \
        exp/norm/check \
        exp/spdy/check \
        exp/sql/check \
@@ -3594,7 +3596,6 @@ TEST_PACKAGES = \
        old/netchan/check \
        old/regexp/check \
        old/template/check \
-       $(os_inotify_check) \
        os/user/check \
        os/signal/check \
        path/filepath/check \
index 8b6bb72..f7c293a 100644 (file)
@@ -699,10 +699,15 @@ toolexeclibgoencoding_DATA = \
        encoding/hex.gox \
        encoding/pem.gox
 
+@LIBGO_IS_LINUX_FALSE@exp_inotify_gox = 
+
+# exp_inotify_gox = exp/inotify.gox
+@LIBGO_IS_LINUX_TRUE@exp_inotify_gox = 
 toolexeclibgoexpdir = $(toolexeclibgodir)/exp
 toolexeclibgoexp_DATA = \
        exp/ebnf.gox \
        exp/gui.gox \
+       $(exp_inotify_gox) \
        exp/norm.gox \
        exp/spdy.gox \
        exp/sql.gox \
@@ -781,12 +786,7 @@ toolexeclibgoold_DATA = \
        old/template.gox
 
 toolexeclibgoosdir = $(toolexeclibgodir)/os
-@LIBGO_IS_LINUX_FALSE@os_inotify_gox = 
-
-# os_inotify_gox = os/inotify.gox
-@LIBGO_IS_LINUX_TRUE@os_inotify_gox = 
 toolexeclibgoos_DATA = \
-       $(os_inotify_gox) \
        os/user.gox \
        os/signal.gox
 
@@ -1579,6 +1579,9 @@ go_exp_ebnf_files = \
 go_exp_gui_files = \
        go/exp/gui/gui.go
 
+go_exp_inotify_files = \
+       go/exp/inotify/inotify_linux.go
+
 go_exp_norm_files = \
        go/exp/norm/composition.go \
        go/exp/norm/forminfo.go \
@@ -1599,11 +1602,13 @@ go_exp_sql_files = \
 
 go_exp_ssh_files = \
        go/exp/ssh/channel.go \
+       go/exp/ssh/client.go \
        go/exp/ssh/common.go \
        go/exp/ssh/doc.go \
        go/exp/ssh/messages.go \
        go/exp/ssh/server.go \
        go/exp/ssh/server_shell.go \
+       go/exp/ssh/session.go \
        go/exp/ssh/transport.go
 
 go_exp_terminal_files = \
@@ -1773,9 +1778,6 @@ go_old_template_files = \
        go/old/template/format.go \
        go/old/template/parse.go
 
-go_os_inotify_files = \
-       go/os/inotify/inotify_linux.go
-
 go_os_user_files = \
        go/os/user/user.go \
        go/os/user/lookup_unix.go
@@ -2171,10 +2173,10 @@ BUILDGOX = \
        f=`echo $< | sed -e 's/.lo$$/.o/'`; \
        $(OBJCOPY) -j .go_export $$f $@.tmp && mv -f $@.tmp $@
 
-@LIBGO_IS_LINUX_FALSE@os_inotify_check = 
+@LIBGO_IS_LINUX_FALSE@exp_inotify_check = 
 
-# os_inotify_check = os/inotify/check
-@LIBGO_IS_LINUX_TRUE@os_inotify_check = 
+# exp_inotify_check = exp/inotify/check
+@LIBGO_IS_LINUX_TRUE@exp_inotify_check = 
 TEST_PACKAGES = \
        asn1/check \
        big/check \
@@ -2273,6 +2275,7 @@ TEST_PACKAGES = \
        encoding/hex/check \
        encoding/pem/check \
        exp/ebnf/check \
+       $(exp_inotify_check) \
        exp/norm/check \
        exp/spdy/check \
        exp/sql/check \
@@ -2304,7 +2307,6 @@ TEST_PACKAGES = \
        old/netchan/check \
        old/regexp/check \
        old/template/check \
-       $(os_inotify_check) \
        os/user/check \
        os/signal/check \
        path/filepath/check \
@@ -5326,6 +5328,13 @@ exp/gui/x11/check: $(CHECK_DEPS)
        @$(CHECK)
 .PHONY: exp/gui/x11/check
 
+exp/inotify.lo: $(go_exp_inotify_files) fmt.gox os.gox strings.gox syscall.gox
+       $(BUILDPACKAGE)
+exp/inotify/check: $(CHECK_DEPS)
+       @$(MKDIR_P) exp/inotify
+       @$(CHECK)
+.PHONY: exp/inotify/check
+
 exp/sql/driver.lo: $(go_exp_sql_driver_files) fmt.gox os.gox reflect.gox \
                strconv.gox
        $(BUILDPACKAGE)
@@ -5601,13 +5610,6 @@ old/template/check: $(CHECK_DEPS)
        @$(CHECK)
 .PHONY: old/template/check
 
-os/inotify.lo: $(go_os_inotify_files) fmt.gox os.gox strings.gox syscall.gox
-       $(BUILDPACKAGE)
-os/inotify/check: $(CHECK_DEPS)
-       @$(MKDIR_P) os/inotify
-       @$(CHECK)
-.PHONY: os/inotify/check
-
 os/user.lo: $(go_os_user_files) fmt.gox os.gox runtime.gox strconv.gox \
                strings.gox syscall.gox
        $(BUILDPACKAGE)
@@ -5929,6 +5931,8 @@ exp/ebnf.gox: exp/ebnf.lo
        $(BUILDGOX)
 exp/gui.gox: exp/gui.lo
        $(BUILDGOX)
+exp/inotify.gox: exp/inotify.lo
+       $(BUILDGOX)
 exp/norm.gox: exp/norm.lo
        $(BUILDGOX)
 exp/spdy.gox: exp/spdy.lo
@@ -6022,8 +6026,6 @@ old/regexp.gox: old/regexp.lo
 old/template.gox: old/template.lo
        $(BUILDGOX)
 
-os/inotify.gox: os/inotify.lo
-       $(BUILDGOX)
 os/user.gox: os/user.lo
        $(BUILDGOX)
 os/signal.gox: os/signal.lo
index 9e1d1ae..b0dde1e 100644 (file)
@@ -58,22 +58,24 @@ func NewInt(x int64) *Int {
 
 // Set sets z to x and returns z.
 func (z *Int) Set(x *Int) *Int {
-       z.abs = z.abs.set(x.abs)
-       z.neg = x.neg
+       if z != x {
+               z.abs = z.abs.set(x.abs)
+               z.neg = x.neg
+       }
        return z
 }
 
 // Abs sets z to |x| (the absolute value of x) and returns z.
 func (z *Int) Abs(x *Int) *Int {
-       z.abs = z.abs.set(x.abs)
+       z.Set(x)
        z.neg = false
        return z
 }
 
 // Neg sets z to -x and returns z.
 func (z *Int) Neg(x *Int) *Int {
-       z.abs = z.abs.set(x.abs)
-       z.neg = len(z.abs) > 0 && !x.neg // 0 has no sign
+       z.Set(x)
+       z.neg = len(z.abs) > 0 && !z.neg // 0 has no sign
        return z
 }
 
@@ -174,7 +176,7 @@ func (z *Int) Quo(x, y *Int) *Int {
 // If y == 0, a division-by-zero run-time panic occurs.
 // Rem implements truncated modulus (like Go); see QuoRem for more details.
 func (z *Int) Rem(x, y *Int) *Int {
-       _, z.abs = nat(nil).div(z.abs, x.abs, y.abs)
+       _, z.abs = nat{}.div(z.abs, x.abs, y.abs)
        z.neg = len(z.abs) > 0 && x.neg // 0 has no sign
        return z
 }
@@ -422,8 +424,8 @@ func (x *Int) Format(s fmt.State, ch int) {
 // scan sets z to the integer value corresponding to the longest possible prefix
 // read from r representing a signed integer number in a given conversion base.
 // It returns z, the actual conversion base used, and an error, if any. In the
-// error case, the value of z is undefined. The syntax follows the syntax of
-// integer literals in Go.
+// error case, the value of z is undefined but the returned value is nil. The
+// syntax follows the syntax of integer literals in Go.
 //
 // The base argument must be 0 or a value from 2 through MaxBase. If the base
 // is 0, the string prefix determines the actual conversion base. A prefix of
@@ -434,7 +436,7 @@ func (z *Int) scan(r io.RuneScanner, base int) (*Int, int, os.Error) {
        // determine sign
        ch, _, err := r.ReadRune()
        if err != nil {
-               return z, 0, err
+               return nil, 0, err
        }
        neg := false
        switch ch {
@@ -448,7 +450,7 @@ func (z *Int) scan(r io.RuneScanner, base int) (*Int, int, os.Error) {
        // determine mantissa
        z.abs, base, err = z.abs.scan(r, base)
        if err != nil {
-               return z, base, err
+               return nil, base, err
        }
        z.neg = len(z.abs) > 0 && neg // 0 has no sign
 
@@ -497,7 +499,7 @@ func (x *Int) Int64() int64 {
 
 // SetString sets z to the value of s, interpreted in the given base,
 // and returns z and a boolean indicating success. If SetString fails,
-// the value of z is undefined.
+// the value of z is undefined but the returned value is nil.
 //
 // The base argument must be 0 or a value from 2 through MaxBase. If the base
 // is 0, the string prefix determines the actual conversion base. A prefix of
@@ -508,10 +510,13 @@ func (z *Int) SetString(s string, base int) (*Int, bool) {
        r := strings.NewReader(s)
        _, _, err := z.scan(r, base)
        if err != nil {
-               return z, false
+               return nil, false
        }
        _, _, err = r.ReadRune()
-       return z, err == os.EOF // err == os.EOF => scan consumed all of s
+       if err != os.EOF {
+               return nil, false
+       }
+       return z, true // err == os.EOF => scan consumed all of s
 }
 
 // SetBytes interprets buf as the bytes of a big-endian unsigned
index b2e1692..fde19c2 100644 (file)
@@ -311,7 +311,16 @@ func TestSetString(t *testing.T) {
                        t.Errorf("#%d (input '%s') ok incorrect (should be %t)", i, test.in, test.ok)
                        continue
                }
-               if !ok1 || !ok2 {
+               if !ok1 {
+                       if n1 != nil {
+                               t.Errorf("#%d (input '%s') n1 != nil", i, test.in)
+                       }
+                       continue
+               }
+               if !ok2 {
+                       if n2 != nil {
+                               t.Errorf("#%d (input '%s') n2 != nil", i, test.in)
+                       }
                        continue
                }
 
index 33d6bb1..c0769d8 100644 (file)
@@ -35,7 +35,7 @@ import (
 // During arithmetic operations, denormalized values may occur but are
 // always normalized before returning the final result. The normalized
 // representation of 0 is the empty or nil slice (length = 0).
-
+//
 type nat []Word
 
 var (
@@ -447,10 +447,10 @@ func (z nat) mulRange(a, b uint64) nat {
        case a == b:
                return z.setUint64(a)
        case a+1 == b:
-               return z.mul(nat(nil).setUint64(a), nat(nil).setUint64(b))
+               return z.mul(nat{}.setUint64(a), nat{}.setUint64(b))
        }
        m := (a + b) / 2
-       return z.mul(nat(nil).mulRange(a, m), nat(nil).mulRange(m+1, b))
+       return z.mul(nat{}.mulRange(a, m), nat{}.mulRange(m+1, b))
 }
 
 // q = (x-r)/y, with 0 <= r < y
@@ -589,7 +589,6 @@ func (x nat) bitLen() int {
 // MaxBase is the largest number base accepted for string conversions.
 const MaxBase = 'z' - 'a' + 10 + 1 // = hexValue('z') + 1
 
-
 func hexValue(ch int) Word {
        d := MaxBase + 1 // illegal base
        switch {
@@ -786,7 +785,7 @@ func (x nat) string(charset string) string {
        }
 
        // preserve x, create local copy for use in repeated divisions
-       q := nat(nil).set(x)
+       q := nat{}.set(x)
        var r Word
 
        // convert
@@ -1192,11 +1191,11 @@ func (n nat) probablyPrime(reps int) bool {
                return false
        }
 
-       nm1 := nat(nil).sub(n, natOne)
+       nm1 := nat{}.sub(n, natOne)
        // 1<<k * q = nm1;
        q, k := nm1.powersOfTwoDecompose()
 
-       nm3 := nat(nil).sub(nm1, natTwo)
+       nm3 := nat{}.sub(nm1, natTwo)
        rand := rand.New(rand.NewSource(int64(n[0])))
 
        var x, y, quotient nat
index 71d0860..4f57328 100644 (file)
@@ -67,7 +67,7 @@ var prodNN = []argNN{
 
 func TestSet(t *testing.T) {
        for _, a := range sumNN {
-               z := nat(nil).set(a.z)
+               z := nat{}.set(a.z)
                if z.cmp(a.z) != 0 {
                        t.Errorf("got z = %v; want %v", z, a.z)
                }
@@ -129,7 +129,7 @@ var mulRangesN = []struct {
 
 func TestMulRangeN(t *testing.T) {
        for i, r := range mulRangesN {
-               prod := nat(nil).mulRange(r.a, r.b).decimalString()
+               prod := nat{}.mulRange(r.a, r.b).decimalString()
                if prod != r.prod {
                        t.Errorf("#%d: got %s; want %s", i, prod, r.prod)
                }
@@ -175,7 +175,7 @@ func toString(x nat, charset string) string {
        s := make([]byte, i)
 
        // don't destroy x
-       q := nat(nil).set(x)
+       q := nat{}.set(x)
 
        // convert
        for len(q) > 0 {
@@ -212,7 +212,7 @@ func TestString(t *testing.T) {
                        t.Errorf("string%+v\n\tgot s = %s; want %s", a, s, a.s)
                }
 
-               x, b, err := nat(nil).scan(strings.NewReader(a.s), len(a.c))
+               x, b, err := nat{}.scan(strings.NewReader(a.s), len(a.c))
                if x.cmp(a.x) != 0 {
                        t.Errorf("scan%+v\n\tgot z = %v; want %v", a, x, a.x)
                }
@@ -271,7 +271,7 @@ var natScanTests = []struct {
 func TestScanBase(t *testing.T) {
        for _, a := range natScanTests {
                r := strings.NewReader(a.s)
-               x, b, err := nat(nil).scan(r, a.base)
+               x, b, err := nat{}.scan(r, a.base)
                if err == nil && !a.ok {
                        t.Errorf("scan%+v\n\texpected error", a)
                }
@@ -651,17 +651,17 @@ var expNNTests = []struct {
 
 func TestExpNN(t *testing.T) {
        for i, test := range expNNTests {
-               x, _, _ := nat(nil).scan(strings.NewReader(test.x), 0)
-               y, _, _ := nat(nil).scan(strings.NewReader(test.y), 0)
-               out, _, _ := nat(nil).scan(strings.NewReader(test.out), 0)
+               x, _, _ := nat{}.scan(strings.NewReader(test.x), 0)
+               y, _, _ := nat{}.scan(strings.NewReader(test.y), 0)
+               out, _, _ := nat{}.scan(strings.NewReader(test.out), 0)
 
                var m nat
 
                if len(test.m) > 0 {
-                       m, _, _ = nat(nil).scan(strings.NewReader(test.m), 0)
+                       m, _, _ = nat{}.scan(strings.NewReader(test.m), 0)
                }
 
-               z := nat(nil).expNN(x, y, m)
+               z := nat{}.expNN(x, y, m)
                if z.cmp(out) != 0 {
                        t.Errorf("#%d got %v want %v", i, z, out)
                }
index f435e63..6b86062 100644 (file)
@@ -13,11 +13,11 @@ import (
        "strings"
 )
 
-// A Rat represents a quotient a/b of arbitrary precision. The zero value for
-// a Rat, 0/0, is not a legal Rat.
+// A Rat represents a quotient a/b of arbitrary precision.
+// The zero value for a Rat represents the value 0.
 type Rat struct {
        a Int
-       b nat
+       b nat // len(b) == 0 acts like b == 1
 }
 
 // NewRat creates a new Rat with numerator a and denominator b.
@@ -29,8 +29,11 @@ func NewRat(a, b int64) *Rat {
 func (z *Rat) SetFrac(a, b *Int) *Rat {
        z.a.neg = a.neg != b.neg
        babs := b.abs
+       if len(babs) == 0 {
+               panic("division by zero")
+       }
        if &z.a == b || alias(z.a.abs, babs) {
-               babs = nat(nil).set(babs) // make a copy
+               babs = nat{}.set(babs) // make a copy
        }
        z.a.abs = z.a.abs.set(a.abs)
        z.b = z.b.set(babs)
@@ -40,6 +43,9 @@ func (z *Rat) SetFrac(a, b *Int) *Rat {
 // SetFrac64 sets z to a/b and returns z.
 func (z *Rat) SetFrac64(a, b int64) *Rat {
        z.a.SetInt64(a)
+       if b == 0 {
+               panic("division by zero")
+       }
        if b < 0 {
                b = -b
                z.a.neg = !z.a.neg
@@ -51,14 +57,55 @@ func (z *Rat) SetFrac64(a, b int64) *Rat {
 // SetInt sets z to x (by making a copy of x) and returns z.
 func (z *Rat) SetInt(x *Int) *Rat {
        z.a.Set(x)
-       z.b = z.b.setWord(1)
+       z.b = z.b.make(0)
        return z
 }
 
 // SetInt64 sets z to x and returns z.
 func (z *Rat) SetInt64(x int64) *Rat {
        z.a.SetInt64(x)
-       z.b = z.b.setWord(1)
+       z.b = z.b.make(0)
+       return z
+}
+
+// Set sets z to x (by making a copy of x) and returns z.
+func (z *Rat) Set(x *Rat) *Rat {
+       if z != x {
+               z.a.Set(&x.a)
+               z.b = z.b.set(x.b)
+       }
+       return z
+}
+
+// Abs sets z to |x| (the absolute value of x) and returns z.
+func (z *Rat) Abs(x *Rat) *Rat {
+       z.Set(x)
+       z.a.neg = false
+       return z
+}
+
+// Neg sets z to -x and returns z.
+func (z *Rat) Neg(x *Rat) *Rat {
+       z.Set(x)
+       z.a.neg = len(z.a.abs) > 0 && !z.a.neg // 0 has no sign
+       return z
+}
+
+// Inv sets z to 1/x and returns z.
+func (z *Rat) Inv(x *Rat) *Rat {
+       if len(x.a.abs) == 0 {
+               panic("division by zero")
+       }
+       z.Set(x)
+       a := z.b
+       if len(a) == 0 {
+               a = a.setWord(1) // materialize numerator
+       }
+       b := z.a.abs
+       if b.cmp(natOne) == 0 {
+               b = b.make(0) // normalize denominator
+       }
+       z.a.abs, z.b = a, b // sign doesn't change
        return z
 }
 
@@ -74,21 +121,24 @@ func (x *Rat) Sign() int {
 
 // IsInt returns true if the denominator of x is 1.
 func (x *Rat) IsInt() bool {
-       return len(x.b) == 1 && x.b[0] == 1
+       return len(x.b) == 0 || x.b.cmp(natOne) == 0
 }
 
-// Num returns the numerator of z; it may be <= 0.
-// The result is a reference to z's numerator; it
-// may change if a new value is assigned to z.
-func (z *Rat) Num() *Int {
-       return &z.a
+// Num returns the numerator of x; it may be <= 0.
+// The result is a reference to x's numerator; it
+// may change if a new value is assigned to x.
+func (x *Rat) Num() *Int {
+       return &x.a
 }
 
-// Denom returns the denominator of z; it is always > 0.
-// The result is a reference to z's denominator; it
-// may change if a new value is assigned to z.
-func (z *Rat) Denom() *Int {
-       return &Int{false, z.b}
+// Denom returns the denominator of x; it is always > 0.
+// The result is a reference to x's denominator; it
+// may change if a new value is assigned to x.
+func (x *Rat) Denom() *Int {
+       if len(x.b) == 0 {
+               return &Int{abs: nat{1}}
+       }
+       return &Int{abs: x.b}
 }
 
 func gcd(x, y nat) nat {
@@ -106,24 +156,47 @@ func gcd(x, y nat) nat {
 }
 
 func (z *Rat) norm() *Rat {
-       f := gcd(z.a.abs, z.b)
-       if len(z.a.abs) == 0 {
-               // z == 0
-               z.a.neg = false // normalize sign
-               z.b = z.b.setWord(1)
-               return z
-       }
-       if f.cmp(natOne) != 0 {
-               z.a.abs, _ = z.a.abs.div(nil, z.a.abs, f)
-               z.b, _ = z.b.div(nil, z.b, f)
+       switch {
+       case len(z.a.abs) == 0:
+               // z == 0 - normalize sign and denominator
+               z.a.neg = false
+               z.b = z.b.make(0)
+       case len(z.b) == 0:
+               // z is normalized int - nothing to do
+       case z.b.cmp(natOne) == 0:
+               // z is int - normalize denominator
+               z.b = z.b.make(0)
+       default:
+               if f := gcd(z.a.abs, z.b); f.cmp(natOne) != 0 {
+                       z.a.abs, _ = z.a.abs.div(nil, z.a.abs, f)
+                       z.b, _ = z.b.div(nil, z.b, f)
+               }
        }
        return z
 }
 
-func mulNat(x *Int, y nat) *Int {
+// mulDenom sets z to the denominator product x*y (by taking into
+// account that 0 values for x or y must be interpreted as 1) and
+// returns z.
+func mulDenom(z, x, y nat) nat {
+       switch {
+       case len(x) == 0:
+               return z.set(y)
+       case len(y) == 0:
+               return z.set(x)
+       }
+       return z.mul(x, y)
+}
+
+// scaleDenom computes x*f.
+// If f == 0 (zero value of denominator), the result is (a copy of) x.
+func scaleDenom(x *Int, f nat) *Int {
        var z Int
-       z.abs = z.abs.mul(x.abs, y)
-       z.neg = len(z.abs) > 0 && x.neg
+       if len(f) == 0 {
+               return z.Set(x)
+       }
+       z.abs = z.abs.mul(x.abs, f)
+       z.neg = x.neg
        return &z
 }
 
@@ -133,39 +206,32 @@ func mulNat(x *Int, y nat) *Int {
 //    0 if x == y
 //   +1 if x >  y
 //
-func (x *Rat) Cmp(y *Rat) (r int) {
-       return mulNat(&x.a, y.b).Cmp(mulNat(&y.a, x.b))
-}
-
-// Abs sets z to |x| (the absolute value of x) and returns z.
-func (z *Rat) Abs(x *Rat) *Rat {
-       z.a.Abs(&x.a)
-       z.b = z.b.set(x.b)
-       return z
+func (x *Rat) Cmp(y *Rat) int {
+       return scaleDenom(&x.a, y.b).Cmp(scaleDenom(&y.a, x.b))
 }
 
 // Add sets z to the sum x+y and returns z.
 func (z *Rat) Add(x, y *Rat) *Rat {
-       a1 := mulNat(&x.a, y.b)
-       a2 := mulNat(&y.a, x.b)
+       a1 := scaleDenom(&x.a, y.b)
+       a2 := scaleDenom(&y.a, x.b)
        z.a.Add(a1, a2)
-       z.b = z.b.mul(x.b, y.b)
+       z.b = mulDenom(z.b, x.b, y.b)
        return z.norm()
 }
 
 // Sub sets z to the difference x-y and returns z.
 func (z *Rat) Sub(x, y *Rat) *Rat {
-       a1 := mulNat(&x.a, y.b)
-       a2 := mulNat(&y.a, x.b)
+       a1 := scaleDenom(&x.a, y.b)
+       a2 := scaleDenom(&y.a, x.b)
        z.a.Sub(a1, a2)
-       z.b = z.b.mul(x.b, y.b)
+       z.b = mulDenom(z.b, x.b, y.b)
        return z.norm()
 }
 
 // Mul sets z to the product x*y and returns z.
 func (z *Rat) Mul(x, y *Rat) *Rat {
        z.a.Mul(&x.a, &y.a)
-       z.b = z.b.mul(x.b, y.b)
+       z.b = mulDenom(z.b, x.b, y.b)
        return z.norm()
 }
 
@@ -175,28 +241,14 @@ func (z *Rat) Quo(x, y *Rat) *Rat {
        if len(y.a.abs) == 0 {
                panic("division by zero")
        }
-       a := mulNat(&x.a, y.b)
-       b := mulNat(&y.a, x.b)
+       a := scaleDenom(&x.a, y.b)
+       b := scaleDenom(&y.a, x.b)
        z.a.abs = a.abs
        z.b = b.abs
        z.a.neg = a.neg != b.neg
        return z.norm()
 }
 
-// Neg sets z to -x (by making a copy of x if necessary) and returns z.
-func (z *Rat) Neg(x *Rat) *Rat {
-       z.a.Neg(&x.a)
-       z.b = z.b.set(x.b)
-       return z
-}
-
-// Set sets z to x (by making a copy of x if necessary) and returns z.
-func (z *Rat) Set(x *Rat) *Rat {
-       z.a.Set(&x.a)
-       z.b = z.b.set(x.b)
-       return z
-}
-
 func ratTok(ch int) bool {
        return strings.IndexRune("+-/0123456789.eE", ch) >= 0
 }
@@ -219,23 +271,23 @@ func (z *Rat) Scan(s fmt.ScanState, ch int) os.Error {
 
 // SetString sets z to the value of s and returns z and a boolean indicating
 // success. s can be given as a fraction "a/b" or as a floating-point number
-// optionally followed by an exponent. If the operation failed, the value of z
-// is undefined.
+// optionally followed by an exponent. If the operation failed, the value of
+// z is undefined but the returned value is nil.
 func (z *Rat) SetString(s string) (*Rat, bool) {
        if len(s) == 0 {
-               return z, false
+               return nil, false
        }
 
        // check for a quotient
        sep := strings.Index(s, "/")
        if sep >= 0 {
                if _, ok := z.a.SetString(s[0:sep], 10); !ok {
-                       return z, false
+                       return nil, false
                }
                s = s[sep+1:]
                var err os.Error
                if z.b, _, err = z.b.scan(strings.NewReader(s), 10); err != nil {
-                       return z, false
+                       return nil, false
                }
                return z.norm(), true
        }
@@ -248,10 +300,10 @@ func (z *Rat) SetString(s string) (*Rat, bool) {
        if e >= 0 {
                if e < sep {
                        // The E must come after the decimal point.
-                       return z, false
+                       return nil, false
                }
                if _, ok := exp.SetString(s[e+1:], 10); !ok {
-                       return z, false
+                       return nil, false
                }
                s = s[0:e]
        }
@@ -261,7 +313,7 @@ func (z *Rat) SetString(s string) (*Rat, bool) {
        }
 
        if _, ok := z.a.SetString(s, 10); !ok {
-               return z, false
+               return nil, false
        }
        powTen := nat{}.expNN(natTen, exp.abs, nil)
        if exp.neg {
@@ -269,7 +321,7 @@ func (z *Rat) SetString(s string) (*Rat, bool) {
                z.norm()
        } else {
                z.a.abs = z.a.abs.mul(z.a.abs, powTen)
-               z.b = z.b.setWord(1)
+               z.b = z.b.make(0)
        }
 
        return z, true
@@ -277,7 +329,11 @@ func (z *Rat) SetString(s string) (*Rat, bool) {
 
 // String returns a string representation of z in the form "a/b" (even if b == 1).
 func (z *Rat) String() string {
-       return z.a.String() + "/" + z.b.decimalString()
+       s := "/1"
+       if len(z.b) != 0 {
+               s = "/" + z.b.decimalString()
+       }
+       return z.a.String() + s
 }
 
 // RatString returns a string representation of z in the form "a/b" if b != 1,
@@ -299,6 +355,7 @@ func (z *Rat) FloatString(prec int) string {
                }
                return s
        }
+       // z.b != 0
 
        q, r := nat{}.div(nat{}, z.a.abs, z.b)
 
index a2b9055..a95e5fe 100644 (file)
@@ -11,6 +11,46 @@ import (
        "testing"
 )
 
+func TestZeroRat(t *testing.T) {
+       var x, y, z Rat
+       y.SetFrac64(0, 42)
+
+       if x.Cmp(&y) != 0 {
+               t.Errorf("x and y should be both equal and zero")
+       }
+
+       if s := x.String(); s != "0/1" {
+               t.Errorf("got x = %s, want 0/1", s)
+       }
+
+       if s := x.RatString(); s != "0" {
+               t.Errorf("got x = %s, want 0", s)
+       }
+
+       z.Add(&x, &y)
+       if s := z.RatString(); s != "0" {
+               t.Errorf("got x+y = %s, want 0", s)
+       }
+
+       z.Sub(&x, &y)
+       if s := z.RatString(); s != "0" {
+               t.Errorf("got x-y = %s, want 0", s)
+       }
+
+       z.Mul(&x, &y)
+       if s := z.RatString(); s != "0" {
+               t.Errorf("got x*y = %s, want 0", s)
+       }
+
+       // check for division by zero
+       defer func() {
+               if s := recover(); s == nil || s.(string) != "division by zero" {
+                       panic(s)
+               }
+       }()
+       z.Quo(&x, &y)
+}
+
 var setStringTests = []struct {
        in, out string
        ok      bool
@@ -50,8 +90,14 @@ func TestRatSetString(t *testing.T) {
        for i, test := range setStringTests {
                x, ok := new(Rat).SetString(test.in)
 
-               if ok != test.ok || ok && x.RatString() != test.out {
-                       t.Errorf("#%d got %s want %s", i, x.RatString(), test.out)
+               if ok {
+                       if !test.ok {
+                               t.Errorf("#%d SetString(%q) expected failure", i, test.in)
+                       } else if x.RatString() != test.out {
+                               t.Errorf("#%d SetString(%q) got %s want %s", i, test.in, x.RatString(), test.out)
+                       }
+               } else if x != nil {
+                       t.Errorf("#%d SetString(%q) got %p want nil", i, test.in, x)
                }
        }
 }
@@ -113,8 +159,10 @@ func TestFloatString(t *testing.T) {
 func TestRatSign(t *testing.T) {
        zero := NewRat(0, 1)
        for _, a := range setStringTests {
-               var x Rat
-               x.SetString(a.in)
+               x, ok := new(Rat).SetString(a.in)
+               if !ok {
+                       continue
+               }
                s := x.Sign()
                e := x.Cmp(zero)
                if s != e {
@@ -153,29 +201,65 @@ func TestRatCmp(t *testing.T) {
 func TestIsInt(t *testing.T) {
        one := NewInt(1)
        for _, a := range setStringTests {
-               var x Rat
-               x.SetString(a.in)
+               x, ok := new(Rat).SetString(a.in)
+               if !ok {
+                       continue
+               }
                i := x.IsInt()
                e := x.Denom().Cmp(one) == 0
                if i != e {
-                       t.Errorf("got %v; want %v for z = %v", i, e, &x)
+                       t.Errorf("got IsInt(%v) == %v; want %v", x, i, e)
                }
        }
 }
 
 func TestRatAbs(t *testing.T) {
-       zero := NewRat(0, 1)
+       zero := new(Rat)
        for _, a := range setStringTests {
-               var z Rat
-               z.SetString(a.in)
-               var e Rat
-               e.Set(&z)
+               x, ok := new(Rat).SetString(a.in)
+               if !ok {
+                       continue
+               }
+               e := new(Rat).Set(x)
                if e.Cmp(zero) < 0 {
-                       e.Sub(zero, &e)
+                       e.Sub(zero, e)
+               }
+               z := new(Rat).Abs(x)
+               if z.Cmp(e) != 0 {
+                       t.Errorf("got Abs(%v) = %v; want %v", x, z, e)
+               }
+       }
+}
+
+func TestRatNeg(t *testing.T) {
+       zero := new(Rat)
+       for _, a := range setStringTests {
+               x, ok := new(Rat).SetString(a.in)
+               if !ok {
+                       continue
+               }
+               e := new(Rat).Sub(zero, x)
+               z := new(Rat).Neg(x)
+               if z.Cmp(e) != 0 {
+                       t.Errorf("got Neg(%v) = %v; want %v", x, z, e)
+               }
+       }
+}
+
+func TestRatInv(t *testing.T) {
+       zero := new(Rat)
+       for _, a := range setStringTests {
+               x, ok := new(Rat).SetString(a.in)
+               if !ok {
+                       continue
+               }
+               if x.Cmp(zero) == 0 {
+                       continue // avoid division by zero
                }
-               z.Abs(&z)
-               if z.Cmp(&e) != 0 {
-                       t.Errorf("got z = %v; want %v", &z, &e)
+               e := new(Rat).SetFrac(x.Denom(), x.Num())
+               z := new(Rat).Inv(x)
+               if z.Cmp(e) != 0 {
+                       t.Errorf("got Inv(%v) = %v; want %v", x, z, e)
                }
        }
 }
@@ -186,10 +270,10 @@ type ratBinArg struct {
 }
 
 func testRatBin(t *testing.T, i int, name string, f ratBinFun, a ratBinArg) {
-       x, _ := NewRat(0, 1).SetString(a.x)
-       y, _ := NewRat(0, 1).SetString(a.y)
-       z, _ := NewRat(0, 1).SetString(a.z)
-       out := f(NewRat(0, 1), x, y)
+       x, _ := new(Rat).SetString(a.x)
+       y, _ := new(Rat).SetString(a.y)
+       z, _ := new(Rat).SetString(a.z)
+       out := f(new(Rat), x, y)
 
        if out.Cmp(z) != 0 {
                t.Errorf("%s #%d got %s want %s", name, i, out, z)
index 4b8ecc5..73b32e7 100644 (file)
@@ -928,11 +928,11 @@ func CreateCertificate(rand io.Reader, template, parent *Certificate, pub *rsa.P
                return
        }
 
-       asn1Issuer, err := asn1.Marshal(parent.Issuer.ToRDNSequence())
+       asn1Issuer, err := asn1.Marshal(parent.Subject.ToRDNSequence())
        if err != nil {
                return
        }
-       asn1Subject, err := asn1.Marshal(parent.Subject.ToRDNSequence())
+       asn1Subject, err := asn1.Marshal(template.Subject.ToRDNSequence())
        if err != nil {
                return
        }
index dbc5273..d113f85 100644 (file)
@@ -6,8 +6,8 @@ package x509
 
 import (
        "asn1"
-       "bytes"
        "big"
+       "bytes"
        "crypto/dsa"
        "crypto/rand"
        "crypto/rsa"
@@ -243,10 +243,11 @@ func TestCreateSelfSignedCertificate(t *testing.T) {
                return
        }
 
+       commonName := "test.example.com"
        template := Certificate{
                SerialNumber: big.NewInt(1),
                Subject: pkix.Name{
-                       CommonName:   "test.example.com",
+                       CommonName:   commonName,
                        Organization: []string{"Acme Co"},
                },
                NotBefore: time.SecondsToUTC(1000),
@@ -283,6 +284,14 @@ func TestCreateSelfSignedCertificate(t *testing.T) {
                t.Errorf("Failed to parse name constraints: %#v", cert.PermittedDNSDomains)
        }
 
+       if cert.Subject.CommonName != commonName {
+               t.Errorf("Subject wasn't correctly copied from the template. Got %s, want %s", cert.Subject.CommonName, commonName)
+       }
+
+       if cert.Issuer.CommonName != commonName {
+               t.Errorf("Issuer wasn't correctly copied from the template. Got %s, want %s", cert.Issuer.CommonName, commonName)
+       }
+
        err = cert.CheckSignatureFrom(cert)
        if err != nil {
                t.Errorf("Signature verification failed: %s", err)
similarity index 100%
rename from libgo/go/os/inotify/inotify_linux_test.go
rename to libgo/go/exp/inotify/inotify_linux_test.go
index aa72604..a6bb46f 100644 (file)
@@ -6,8 +6,8 @@ package inotify
 
 import (
        "os"
-       "time"
        "testing"
+       "time"
 )
 
 func TestInotifyEvents(t *testing.T) {
index 922584f..f69b735 100644 (file)
@@ -68,7 +68,7 @@ type channel struct {
        weClosed    bool
        dead        bool
 
-       serverConn            *ServerConnection
+       serverConn            *ServerConn
        myId, theirId         uint32
        myWindow, theirWindow uint32
        maxPacketSize         uint32
diff --git a/libgo/go/exp/ssh/client.go b/libgo/go/exp/ssh/client.go
new file mode 100644 (file)
index 0000000..3311385
--- /dev/null
@@ -0,0 +1,490 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssh
+
+import (
+       "big"
+       "crypto"
+       "crypto/rand"
+       "fmt"
+       "io"
+       "net"
+       "os"
+       "sync"
+)
+
+// clientVersion is the fixed identification string that the client will use.
+var clientVersion = []byte("SSH-2.0-Go\r\n")
+
+// ClientConn represents the client side of an SSH connection.
+type ClientConn struct {
+       *transport
+       config *ClientConfig
+       chanlist
+}
+
+// Client returns a new SSH client connection using c as the underlying transport.
+func Client(c net.Conn, config *ClientConfig) (*ClientConn, os.Error) {
+       conn := &ClientConn{
+               transport: newTransport(c, config.rand()),
+               config:    config,
+       }
+       if err := conn.handshake(); err != nil {
+               conn.Close()
+               return nil, err
+       }
+       if err := conn.authenticate(); err != nil {
+               conn.Close()
+               return nil, err
+       }
+       go conn.mainLoop()
+       return conn, nil
+}
+
+// handshake performs the client side key exchange. See RFC 4253 Section 7.
+func (c *ClientConn) handshake() os.Error {
+       var magics handshakeMagics
+
+       if _, err := c.Write(clientVersion); err != nil {
+               return err
+       }
+       if err := c.Flush(); err != nil {
+               return err
+       }
+       magics.clientVersion = clientVersion[:len(clientVersion)-2]
+
+       // read remote server version
+       version, err := readVersion(c)
+       if err != nil {
+               return err
+       }
+       magics.serverVersion = version
+       clientKexInit := kexInitMsg{
+               KexAlgos:                supportedKexAlgos,
+               ServerHostKeyAlgos:      supportedHostKeyAlgos,
+               CiphersClientServer:     supportedCiphers,
+               CiphersServerClient:     supportedCiphers,
+               MACsClientServer:        supportedMACs,
+               MACsServerClient:        supportedMACs,
+               CompressionClientServer: supportedCompressions,
+               CompressionServerClient: supportedCompressions,
+       }
+       kexInitPacket := marshal(msgKexInit, clientKexInit)
+       magics.clientKexInit = kexInitPacket
+
+       if err := c.writePacket(kexInitPacket); err != nil {
+               return err
+       }
+       packet, err := c.readPacket()
+       if err != nil {
+               return err
+       }
+
+       magics.serverKexInit = packet
+
+       var serverKexInit kexInitMsg
+       if err = unmarshal(&serverKexInit, packet, msgKexInit); err != nil {
+               return err
+       }
+
+       kexAlgo, hostKeyAlgo, ok := findAgreedAlgorithms(c.transport, &clientKexInit, &serverKexInit)
+       if !ok {
+               return os.NewError("ssh: no common algorithms")
+       }
+
+       if serverKexInit.FirstKexFollows && kexAlgo != serverKexInit.KexAlgos[0] {
+               // The server sent a Kex message for the wrong algorithm,
+               // which we have to ignore.
+               if _, err := c.readPacket(); err != nil {
+                       return err
+               }
+       }
+
+       var H, K []byte
+       var hashFunc crypto.Hash
+       switch kexAlgo {
+       case kexAlgoDH14SHA1:
+               hashFunc = crypto.SHA1
+               dhGroup14Once.Do(initDHGroup14)
+               H, K, err = c.kexDH(dhGroup14, hashFunc, &magics, hostKeyAlgo)
+       default:
+               err = fmt.Errorf("ssh: unexpected key exchange algorithm %v", kexAlgo)
+       }
+       if err != nil {
+               return err
+       }
+
+       if err = c.writePacket([]byte{msgNewKeys}); err != nil {
+               return err
+       }
+       if err = c.transport.writer.setupKeys(clientKeys, K, H, H, hashFunc); err != nil {
+               return err
+       }
+       if packet, err = c.readPacket(); err != nil {
+               return err
+       }
+       if packet[0] != msgNewKeys {
+               return UnexpectedMessageError{msgNewKeys, packet[0]}
+       }
+       return c.transport.reader.setupKeys(serverKeys, K, H, H, hashFunc)
+}
+
+// authenticate authenticates with the remote server. See RFC 4252. 
+// Only "password" authentication is supported.
+func (c *ClientConn) authenticate() os.Error {
+       if err := c.writePacket(marshal(msgServiceRequest, serviceRequestMsg{serviceUserAuth})); err != nil {
+               return err
+       }
+       packet, err := c.readPacket()
+       if err != nil {
+               return err
+       }
+
+       var serviceAccept serviceAcceptMsg
+       if err = unmarshal(&serviceAccept, packet, msgServiceAccept); err != nil {
+               return err
+       }
+
+       // TODO(dfc) support proper authentication method negotation
+       method := "none"
+       if c.config.Password != "" {
+               method = "password"
+       }
+       if err := c.sendUserAuthReq(method); err != nil {
+               return err
+       }
+
+       if packet, err = c.readPacket(); err != nil {
+               return err
+       }
+
+       if packet[0] != msgUserAuthSuccess {
+               return UnexpectedMessageError{msgUserAuthSuccess, packet[0]}
+       }
+       return nil
+}
+
+func (c *ClientConn) sendUserAuthReq(method string) os.Error {
+       length := stringLength([]byte(c.config.Password)) + 1
+       payload := make([]byte, length)
+       // always false for password auth, see RFC 4252 Section 8.
+       payload[0] = 0
+       marshalString(payload[1:], []byte(c.config.Password))
+
+       return c.writePacket(marshal(msgUserAuthRequest, userAuthRequestMsg{
+               User:    c.config.User,
+               Service: serviceSSH,
+               Method:  method,
+               Payload: payload,
+       }))
+}
+
+// kexDH performs Diffie-Hellman key agreement on a ClientConn. The
+// returned values are given the same names as in RFC 4253, section 8.
+func (c *ClientConn) kexDH(group *dhGroup, hashFunc crypto.Hash, magics *handshakeMagics, hostKeyAlgo string) ([]byte, []byte, os.Error) {
+       x, err := rand.Int(c.config.rand(), group.p)
+       if err != nil {
+               return nil, nil, err
+       }
+       X := new(big.Int).Exp(group.g, x, group.p)
+       kexDHInit := kexDHInitMsg{
+               X: X,
+       }
+       if err := c.writePacket(marshal(msgKexDHInit, kexDHInit)); err != nil {
+               return nil, nil, err
+       }
+
+       packet, err := c.readPacket()
+       if err != nil {
+               return nil, nil, err
+       }
+
+       var kexDHReply = new(kexDHReplyMsg)
+       if err = unmarshal(kexDHReply, packet, msgKexDHReply); err != nil {
+               return nil, nil, err
+       }
+
+       if kexDHReply.Y.Sign() == 0 || kexDHReply.Y.Cmp(group.p) >= 0 {
+               return nil, nil, os.NewError("server DH parameter out of bounds")
+       }
+
+       kInt := new(big.Int).Exp(kexDHReply.Y, x, group.p)
+       h := hashFunc.New()
+       writeString(h, magics.clientVersion)
+       writeString(h, magics.serverVersion)
+       writeString(h, magics.clientKexInit)
+       writeString(h, magics.serverKexInit)
+       writeString(h, kexDHReply.HostKey)
+       writeInt(h, X)
+       writeInt(h, kexDHReply.Y)
+       K := make([]byte, intLength(kInt))
+       marshalInt(K, kInt)
+       h.Write(K)
+
+       H := h.Sum()
+
+       return H, K, nil
+}
+
+// openChan opens a new client channel. The most common session type is "session". 
+// The full set of valid session types are listed in RFC 4250 4.9.1.
+func (c *ClientConn) openChan(typ string) (*clientChan, os.Error) {
+       ch := c.newChan(c.transport)
+       if err := c.writePacket(marshal(msgChannelOpen, channelOpenMsg{
+               ChanType:      typ,
+               PeersId:       ch.id,
+               PeersWindow:   1 << 14,
+               MaxPacketSize: 1 << 15, // RFC 4253 6.1
+       })); err != nil {
+               c.chanlist.remove(ch.id)
+               return nil, err
+       }
+       // wait for response
+       switch msg := (<-ch.msg).(type) {
+       case *channelOpenConfirmMsg:
+               ch.peersId = msg.MyId
+       case *channelOpenFailureMsg:
+               c.chanlist.remove(ch.id)
+               return nil, os.NewError(msg.Message)
+       default:
+               c.chanlist.remove(ch.id)
+               return nil, os.NewError("Unexpected packet")
+       }
+       return ch, nil
+}
+
+// mainloop reads incoming messages and routes channel messages
+// to their respective ClientChans.
+func (c *ClientConn) mainLoop() {
+       for {
+               packet, err := c.readPacket()
+               if err != nil {
+                       // TODO(dfc) signal the underlying close to all channels
+                       c.Close()
+                       return
+               }
+               // TODO(dfc) A note on blocking channel use. 
+               // The msg, win, data and dataExt channels of a clientChan can 
+               // cause this loop to block indefinately if the consumer does 
+               // not service them. 
+               switch msg := decode(packet).(type) {
+               case *channelOpenMsg:
+                       c.getChan(msg.PeersId).msg <- msg
+               case *channelOpenConfirmMsg:
+                       c.getChan(msg.PeersId).msg <- msg
+               case *channelOpenFailureMsg:
+                       c.getChan(msg.PeersId).msg <- msg
+               case *channelCloseMsg:
+                       ch := c.getChan(msg.PeersId)
+                       close(ch.win)
+                       close(ch.data)
+                       close(ch.dataExt)
+                       c.chanlist.remove(msg.PeersId)
+               case *channelEOFMsg:
+                       c.getChan(msg.PeersId).msg <- msg
+               case *channelRequestSuccessMsg:
+                       c.getChan(msg.PeersId).msg <- msg
+               case *channelRequestFailureMsg:
+                       c.getChan(msg.PeersId).msg <- msg
+               case *channelRequestMsg:
+                       c.getChan(msg.PeersId).msg <- msg
+               case *windowAdjustMsg:
+                       c.getChan(msg.PeersId).win <- int(msg.AdditionalBytes)
+               case *channelData:
+                       c.getChan(msg.PeersId).data <- msg.Payload
+               case *channelExtendedData:
+                       // RFC 4254 5.2 defines data_type_code 1 to be data destined 
+                       // for stderr on interactive sessions. Other data types are
+                       // silently discarded.
+                       if msg.Datatype == 1 {
+                               c.getChan(msg.PeersId).dataExt <- msg.Payload
+                       }
+               default:
+                       fmt.Printf("mainLoop: unhandled %#v\n", msg)
+               }
+       }
+}
+
+// Dial connects to the given network address using net.Dial and 
+// then initiates a SSH handshake, returning the resulting client connection.
+func Dial(network, addr string, config *ClientConfig) (*ClientConn, os.Error) {
+       conn, err := net.Dial(network, addr)
+       if err != nil {
+               return nil, err
+       }
+       return Client(conn, config)
+}
+
+// A ClientConfig structure is used to configure a ClientConn. After one has 
+// been passed to an SSH function it must not be modified.
+type ClientConfig struct {
+       // Rand provides the source of entropy for key exchange. If Rand is 
+       // nil, the cryptographic random reader in package crypto/rand will 
+       // be used.
+       Rand io.Reader
+
+       // The username to authenticate.
+       User string
+
+       // Used for "password" method authentication.
+       Password string
+}
+
+func (c *ClientConfig) rand() io.Reader {
+       if c.Rand == nil {
+               return rand.Reader
+       }
+       return c.Rand
+}
+
+// A clientChan represents a single RFC 4254 channel that is multiplexed 
+// over a single SSH connection.
+type clientChan struct {
+       packetWriter
+       id, peersId uint32
+       data        chan []byte      // receives the payload of channelData messages
+       dataExt     chan []byte      // receives the payload of channelExtendedData messages
+       win         chan int         // receives window adjustments
+       msg         chan interface{} // incoming messages
+}
+
+func newClientChan(t *transport, id uint32) *clientChan {
+       return &clientChan{
+               packetWriter: t,
+               id:           id,
+               data:         make(chan []byte, 16),
+               dataExt:      make(chan []byte, 16),
+               win:          make(chan int, 16),
+               msg:          make(chan interface{}, 16),
+       }
+}
+
+// Close closes the channel. This does not close the underlying connection.
+func (c *clientChan) Close() os.Error {
+       return c.writePacket(marshal(msgChannelClose, channelCloseMsg{
+               PeersId: c.id,
+       }))
+}
+
+func (c *clientChan) sendChanReq(req channelRequestMsg) os.Error {
+       if err := c.writePacket(marshal(msgChannelRequest, req)); err != nil {
+               return err
+       }
+       msg := <-c.msg
+       if _, ok := msg.(*channelRequestSuccessMsg); ok {
+               return nil
+       }
+       return fmt.Errorf("failed to complete request: %s, %#v", req.Request, msg)
+}
+
+// Thread safe channel list.
+type chanlist struct {
+       // protects concurrent access to chans
+       sync.Mutex
+       // chans are indexed by the local id of the channel, clientChan.id.
+       // The PeersId value of messages received by ClientConn.mainloop is
+       // used to locate the right local clientChan in this slice.
+       chans []*clientChan
+}
+
+// Allocate a new ClientChan with the next avail local id.
+func (c *chanlist) newChan(t *transport) *clientChan {
+       c.Lock()
+       defer c.Unlock()
+       for i := range c.chans {
+               if c.chans[i] == nil {
+                       ch := newClientChan(t, uint32(i))
+                       c.chans[i] = ch
+                       return ch
+               }
+       }
+       i := len(c.chans)
+       ch := newClientChan(t, uint32(i))
+       c.chans = append(c.chans, ch)
+       return ch
+}
+
+func (c *chanlist) getChan(id uint32) *clientChan {
+       c.Lock()
+       defer c.Unlock()
+       return c.chans[int(id)]
+}
+
+func (c *chanlist) remove(id uint32) {
+       c.Lock()
+       defer c.Unlock()
+       c.chans[int(id)] = nil
+}
+
+// A chanWriter represents the stdin of a remote process.
+type chanWriter struct {
+       win          chan int // receives window adjustments
+       id           uint32   // this channel's id
+       rwin         int      // current rwin size
+       packetWriter          // for sending channelDataMsg
+}
+
+// Write writes data to the remote process's standard input.
+func (w *chanWriter) Write(data []byte) (n int, err os.Error) {
+       for {
+               if w.rwin == 0 {
+                       win, ok := <-w.win
+                       if !ok {
+                               return 0, os.EOF
+                       }
+                       w.rwin += win
+                       continue
+               }
+               n = len(data)
+               packet := make([]byte, 0, 9+n)
+               packet = append(packet, msgChannelData,
+                       byte(w.id)>>24, byte(w.id)>>16, byte(w.id)>>8, byte(w.id),
+                       byte(n)>>24, byte(n)>>16, byte(n)>>8, byte(n))
+               err = w.writePacket(append(packet, data...))
+               w.rwin -= n
+               return
+       }
+       panic("unreachable")
+}
+
+func (w *chanWriter) Close() os.Error {
+       return w.writePacket(marshal(msgChannelEOF, channelEOFMsg{w.id}))
+}
+
+// A chanReader represents stdout or stderr of a remote process.
+type chanReader struct {
+       // TODO(dfc) a fixed size channel may not be the right data structure.
+       // If writes to this channel block, they will block mainLoop, making
+       // it unable to receive new messages from the remote side.
+       data         chan []byte // receives data from remote
+       id           uint32
+       packetWriter // for sending windowAdjustMsg
+       buf          []byte
+}
+
+// Read reads data from the remote process's stdout or stderr.
+func (r *chanReader) Read(data []byte) (int, os.Error) {
+       var ok bool
+       for {
+               if len(r.buf) > 0 {
+                       n := copy(data, r.buf)
+                       r.buf = r.buf[n:]
+                       msg := windowAdjustMsg{
+                               PeersId:         r.id,
+                               AdditionalBytes: uint32(n),
+                       }
+                       return n, r.writePacket(marshal(msgChannelWindowAdjust, msg))
+               }
+               r.buf, ok = <-r.data
+               if !ok {
+                       return 0, os.EOF
+               }
+       }
+       panic("unreachable")
+}
+
+func (r *chanReader) Close() os.Error {
+       return r.writePacket(marshal(msgChannelEOF, channelEOFMsg{r.id}))
+}
index 54a7ba9..fc842b0 100644 (file)
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 /*
-Package ssh implements an SSH server.
+Package ssh implements an SSH client and server.
 
 SSH is a transport security protocol, an authentication protocol and a
 family of application protocols. The most typical application level
@@ -11,26 +11,29 @@ protocol is a remote shell and this is specifically implemented.  However,
 the multiplexed nature of SSH is exposed to users that wish to support
 others.
 
-An SSH server is represented by a Server, which manages a number of
-ServerConnections and handles authentication.
+An SSH server is represented by a ServerConfig, which holds certificate
+details and handles authentication of ServerConns.
 
-       var s Server
-       s.PubKeyCallback = pubKeyAuth
-       s.PasswordCallback = passwordAuth
+       config := new(ServerConfig)
+       config.PubKeyCallback = pubKeyAuth
+       config.PasswordCallback = passwordAuth
 
        pemBytes, err := ioutil.ReadFile("id_rsa")
        if err != nil {
                panic("Failed to load private key")
        }
-       err = s.SetRSAPrivateKey(pemBytes)
+       err = config.SetRSAPrivateKey(pemBytes)
        if err != nil {
                panic("Failed to parse private key")
        }
 
-Once a Server has been set up, connections can be attached.
+Once a ServerConfig has been configured, connections can be accepted.
 
-       var sConn ServerConnection
-       sConn.Server = &s
+       listener := Listen("tcp", "0.0.0.0:2022", config)
+       sConn, err := listener.Accept()
+       if err != nil {
+               panic("failed to accept incoming connection")
+       }
        err = sConn.Handshake(conn)
        if err != nil {
                panic("failed to handshake")
@@ -38,7 +41,6 @@ Once a Server has been set up, connections can be attached.
 
 An SSH connection multiplexes several channels, which must be accepted themselves:
 
-
        for {
                channel, err := sConn.Accept()
                if err != nil {
@@ -75,5 +77,29 @@ present a simple terminal interface.
                }
                return
        }()
+
+An SSH client is represented with a ClientConn. Currently only the "password"
+authentication method is supported. 
+
+       config := &ClientConfig{
+               User: "username",
+               Password: "123456",
+       }
+       client, err := Dial("yourserver.com:22", config)
+
+Each ClientConn can support multiple interactive sessions, represented by a Session. 
+
+       session, err := client.NewSession()
+
+Once a Session is created, you can execute a single command on the remote side 
+using the Exec method.
+
+       if err := session.Exec("/usr/bin/whoami"); err != nil {
+               panic("Failed to exec: " + err.String())
+       }
+       reader := bufio.NewReader(session.Stdin)
+       line, _, _ := reader.ReadLine()
+       fmt.Println(line)
+       session.Close()
 */
 package ssh
index 1d0bc57..7771f2b 100644 (file)
@@ -154,7 +154,7 @@ type channelData struct {
 type channelExtendedData struct {
        PeersId  uint32
        Datatype uint32
-       Data     string
+       Payload  []byte `ssh:"rest"`
 }
 
 type channelRequestMsg struct {
index 410cafc..3a640fc 100644 (file)
@@ -10,19 +10,23 @@ import (
        "crypto"
        "crypto/rand"
        "crypto/rsa"
-       _ "crypto/sha1"
        "crypto/x509"
        "encoding/pem"
+       "io"
        "net"
        "os"
        "sync"
 )
 
-// Server represents an SSH server. A Server may have several ServerConnections.
-type Server struct {
+type ServerConfig struct {
        rsa           *rsa.PrivateKey
        rsaSerialized []byte
 
+       // Rand provides the source of entropy for key exchange. If Rand is 
+       // nil, the cryptographic random reader in package crypto/rand will 
+       // be used.
+       Rand io.Reader
+
        // NoClientAuth is true if clients are allowed to connect without
        // authenticating.
        NoClientAuth bool
@@ -38,11 +42,18 @@ type Server struct {
        PubKeyCallback func(user, algo string, pubkey []byte) bool
 }
 
+func (c *ServerConfig) rand() io.Reader {
+       if c.Rand == nil {
+               return rand.Reader
+       }
+       return c.Rand
+}
+
 // SetRSAPrivateKey sets the private key for a Server. A Server must have a
 // private key configured in order to accept connections. The private key must
 // be in the form of a PEM encoded, PKCS#1, RSA private key. The file "id_rsa"
 // typically contains such a key.
-func (s *Server) SetRSAPrivateKey(pemBytes []byte) os.Error {
+func (s *ServerConfig) SetRSAPrivateKey(pemBytes []byte) os.Error {
        block, _ := pem.Decode(pemBytes)
        if block == nil {
                return os.NewError("ssh: no key found")
@@ -109,7 +120,7 @@ func parseRSASig(in []byte) (sig []byte, ok bool) {
 }
 
 // cachedPubKey contains the results of querying whether a public key is
-// acceptable for a user. The cache only applies to a single ServerConnection.
+// acceptable for a user. The cache only applies to a single ServerConn.
 type cachedPubKey struct {
        user, algo string
        pubKey     []byte
@@ -118,11 +129,10 @@ type cachedPubKey struct {
 
 const maxCachedPubKeys = 16
 
-// ServerConnection represents an incomming connection to a Server.
-type ServerConnection struct {
-       Server *Server
-
+// A ServerConn represents an incomming connection.
+type ServerConn struct {
        *transport
+       config *ServerConfig
 
        channels   map[uint32]*channel
        nextChanId uint32
@@ -139,9 +149,20 @@ type ServerConnection struct {
        cachedPubKeys []cachedPubKey
 }
 
+// Server returns a new SSH server connection
+// using c as the underlying transport.
+func Server(c net.Conn, config *ServerConfig) *ServerConn {
+       conn := &ServerConn{
+               transport: newTransport(c, config.rand()),
+               channels:  make(map[uint32]*channel),
+               config:    config,
+       }
+       return conn
+}
+
 // kexDH performs Diffie-Hellman key agreement on a ServerConnection. The
 // returned values are given the same names as in RFC 4253, section 8.
-func (s *ServerConnection) kexDH(group *dhGroup, hashFunc crypto.Hash, magics *handshakeMagics, hostKeyAlgo string) (H, K []byte, err os.Error) {
+func (s *ServerConn) kexDH(group *dhGroup, hashFunc crypto.Hash, magics *handshakeMagics, hostKeyAlgo string) (H, K []byte, err os.Error) {
        packet, err := s.readPacket()
        if err != nil {
                return
@@ -155,7 +176,7 @@ func (s *ServerConnection) kexDH(group *dhGroup, hashFunc crypto.Hash, magics *h
                return nil, nil, os.NewError("client DH parameter out of bounds")
        }
 
-       y, err := rand.Int(rand.Reader, group.p)
+       y, err := rand.Int(s.config.rand(), group.p)
        if err != nil {
                return
        }
@@ -166,7 +187,7 @@ func (s *ServerConnection) kexDH(group *dhGroup, hashFunc crypto.Hash, magics *h
        var serializedHostKey []byte
        switch hostKeyAlgo {
        case hostAlgoRSA:
-               serializedHostKey = s.Server.rsaSerialized
+               serializedHostKey = s.config.rsaSerialized
        default:
                return nil, nil, os.NewError("internal error")
        }
@@ -192,7 +213,7 @@ func (s *ServerConnection) kexDH(group *dhGroup, hashFunc crypto.Hash, magics *h
        var sig []byte
        switch hostKeyAlgo {
        case hostAlgoRSA:
-               sig, err = rsa.SignPKCS1v15(rand.Reader, s.Server.rsa, hashFunc, hh)
+               sig, err = rsa.SignPKCS1v15(s.config.rand(), s.config.rsa, hashFunc, hh)
                if err != nil {
                        return
                }
@@ -257,19 +278,20 @@ func buildDataSignedForAuth(sessionId []byte, req userAuthRequestMsg, algo, pubK
        return ret
 }
 
-// Handshake performs an SSH transport and client authentication on the given ServerConnection.
-func (s *ServerConnection) Handshake(conn net.Conn) os.Error {
+// Handshake performs an SSH transport and client authentication on the given ServerConn.
+func (s *ServerConn) Handshake() os.Error {
        var magics handshakeMagics
-       s.transport = newTransport(conn, rand.Reader)
-
-       if _, err := conn.Write(serverVersion); err != nil {
+       if _, err := s.Write(serverVersion); err != nil {
+               return err
+       }
+       if err := s.Flush(); err != nil {
                return err
        }
        magics.serverVersion = serverVersion[:len(serverVersion)-2]
 
-       version, ok := readVersion(s.transport)
-       if !ok {
-               return os.NewError("failed to read version string from client")
+       version, err := readVersion(s)
+       if err != nil {
+               return err
        }
        magics.clientVersion = version
 
@@ -310,8 +332,7 @@ func (s *ServerConnection) Handshake(conn net.Conn) os.Error {
        if clientKexInit.FirstKexFollows && kexAlgo != clientKexInit.KexAlgos[0] {
                // The client sent a Kex message for the wrong algorithm,
                // which we have to ignore.
-               _, err := s.readPacket()
-               if err != nil {
+               if _, err := s.readPacket(); err != nil {
                        return err
                }
        }
@@ -324,32 +345,27 @@ func (s *ServerConnection) Handshake(conn net.Conn) os.Error {
                dhGroup14Once.Do(initDHGroup14)
                H, K, err = s.kexDH(dhGroup14, hashFunc, &magics, hostKeyAlgo)
        default:
-               err = os.NewError("ssh: internal error")
+               err = os.NewError("ssh: unexpected key exchange algorithm " + kexAlgo)
        }
-
        if err != nil {
                return err
        }
 
-       packet = []byte{msgNewKeys}
-       if err = s.writePacket(packet); err != nil {
+       if err = s.writePacket([]byte{msgNewKeys}); err != nil {
                return err
        }
        if err = s.transport.writer.setupKeys(serverKeys, K, H, H, hashFunc); err != nil {
                return err
        }
-
        if packet, err = s.readPacket(); err != nil {
                return err
        }
+
        if packet[0] != msgNewKeys {
                return UnexpectedMessageError{msgNewKeys, packet[0]}
        }
-
        s.transport.reader.setupKeys(clientKeys, K, H, H, hashFunc)
-
-       packet, err = s.readPacket()
-       if err != nil {
+       if packet, err = s.readPacket(); err != nil {
                return err
        }
 
@@ -360,20 +376,16 @@ func (s *ServerConnection) Handshake(conn net.Conn) os.Error {
        if serviceRequest.Service != serviceUserAuth {
                return os.NewError("ssh: requested service '" + serviceRequest.Service + "' before authenticating")
        }
-
        serviceAccept := serviceAcceptMsg{
                Service: serviceUserAuth,
        }
-       packet = marshal(msgServiceAccept, serviceAccept)
-       if err = s.writePacket(packet); err != nil {
+       if err = s.writePacket(marshal(msgServiceAccept, serviceAccept)); err != nil {
                return err
        }
 
        if err = s.authenticate(H); err != nil {
                return err
        }
-
-       s.channels = make(map[uint32]*channel)
        return nil
 }
 
@@ -382,8 +394,8 @@ func isAcceptableAlgo(algo string) bool {
 }
 
 // testPubKey returns true if the given public key is acceptable for the user.
-func (s *ServerConnection) testPubKey(user, algo string, pubKey []byte) bool {
-       if s.Server.PubKeyCallback == nil || !isAcceptableAlgo(algo) {
+func (s *ServerConn) testPubKey(user, algo string, pubKey []byte) bool {
+       if s.config.PubKeyCallback == nil || !isAcceptableAlgo(algo) {
                return false
        }
 
@@ -393,7 +405,7 @@ func (s *ServerConnection) testPubKey(user, algo string, pubKey []byte) bool {
                }
        }
 
-       result := s.Server.PubKeyCallback(user, algo, pubKey)
+       result := s.config.PubKeyCallback(user, algo, pubKey)
        if len(s.cachedPubKeys) < maxCachedPubKeys {
                c := cachedPubKey{
                        user:   user,
@@ -408,7 +420,7 @@ func (s *ServerConnection) testPubKey(user, algo string, pubKey []byte) bool {
        return result
 }
 
-func (s *ServerConnection) authenticate(H []byte) os.Error {
+func (s *ServerConn) authenticate(H []byte) os.Error {
        var userAuthReq userAuthRequestMsg
        var err os.Error
        var packet []byte
@@ -428,11 +440,11 @@ userAuthLoop:
 
                switch userAuthReq.Method {
                case "none":
-                       if s.Server.NoClientAuth {
+                       if s.config.NoClientAuth {
                                break userAuthLoop
                        }
                case "password":
-                       if s.Server.PasswordCallback == nil {
+                       if s.config.PasswordCallback == nil {
                                break
                        }
                        payload := userAuthReq.Payload
@@ -445,11 +457,11 @@ userAuthLoop:
                                return ParseError{msgUserAuthRequest}
                        }
 
-                       if s.Server.PasswordCallback(userAuthReq.User, string(password)) {
+                       if s.config.PasswordCallback(userAuthReq.User, string(password)) {
                                break userAuthLoop
                        }
                case "publickey":
-                       if s.Server.PubKeyCallback == nil {
+                       if s.config.PubKeyCallback == nil {
                                break
                        }
                        payload := userAuthReq.Payload
@@ -520,10 +532,10 @@ userAuthLoop:
                }
 
                var failureMsg userAuthFailureMsg
-               if s.Server.PasswordCallback != nil {
+               if s.config.PasswordCallback != nil {
                        failureMsg.Methods = append(failureMsg.Methods, "password")
                }
-               if s.Server.PubKeyCallback != nil {
+               if s.config.PubKeyCallback != nil {
                        failureMsg.Methods = append(failureMsg.Methods, "publickey")
                }
 
@@ -546,9 +558,9 @@ userAuthLoop:
 
 const defaultWindowSize = 32768
 
-// Accept reads and processes messages on a ServerConnection. It must be called
+// Accept reads and processes messages on a ServerConn. It must be called
 // in order to demultiplex messages to any resulting Channels.
-func (s *ServerConnection) Accept() (Channel, os.Error) {
+func (s *ServerConn) Accept() (Channel, os.Error) {
        if s.err != nil {
                return nil, s.err
        }
@@ -643,3 +655,44 @@ func (s *ServerConnection) Accept() (Channel, os.Error) {
 
        panic("unreachable")
 }
+
+// A Listener implements a network listener (net.Listener) for SSH connections.
+type Listener struct {
+       listener net.Listener
+       config   *ServerConfig
+}
+
+// Accept waits for and returns the next incoming SSH connection.
+// The receiver should call Handshake() in another goroutine 
+// to avoid blocking the accepter.
+func (l *Listener) Accept() (*ServerConn, os.Error) {
+       c, err := l.listener.Accept()
+       if err != nil {
+               return nil, err
+       }
+       conn := Server(c, l.config)
+       return conn, nil
+}
+
+// Addr returns the listener's network address.
+func (l *Listener) Addr() net.Addr {
+       return l.listener.Addr()
+}
+
+// Close closes the listener.
+func (l *Listener) Close() os.Error {
+       return l.listener.Close()
+}
+
+// Listen creates an SSH listener accepting connections on
+// the given network address using net.Listen.
+func Listen(network, addr string, config *ServerConfig) (*Listener, os.Error) {
+       l, err := net.Listen(network, addr)
+       if err != nil {
+               return nil, err
+       }
+       return &Listener{
+               l,
+               config,
+       }, nil
+}
diff --git a/libgo/go/exp/ssh/session.go b/libgo/go/exp/ssh/session.go
new file mode 100644 (file)
index 0000000..13df2f0
--- /dev/null
@@ -0,0 +1,132 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package ssh
+
+// Session implements an interactive session described in
+// "RFC 4254, section 6".
+
+import (
+       "encoding/binary"
+       "io"
+       "os"
+)
+
+// A Session represents a connection to a remote command or shell.
+type Session struct {
+       // Writes to Stdin are made available to the remote command's standard input.
+       // Closing Stdin causes the command to observe an EOF on its standard input.
+       Stdin io.WriteCloser
+
+       // Reads from Stdout and Stderr consume from the remote command's standard
+       // output and error streams, respectively.
+       // There is a fixed amount of buffering that is shared for the two streams.
+       // Failing to read from either may eventually cause the command to block.
+       // Closing Stdout unblocks such writes and causes them to return errors.
+       Stdout io.ReadCloser
+       Stderr io.Reader
+
+       *clientChan // the channel backing this session
+
+       started bool // started is set to true once a Shell or Exec is invoked.
+}
+
+// Setenv sets an environment variable that will be applied to any
+// command executed by Shell or Exec.
+func (s *Session) Setenv(name, value string) os.Error {
+       n, v := []byte(name), []byte(value)
+       nlen, vlen := stringLength(n), stringLength(v)
+       payload := make([]byte, nlen+vlen)
+       marshalString(payload[:nlen], n)
+       marshalString(payload[nlen:], v)
+
+       return s.sendChanReq(channelRequestMsg{
+               PeersId:             s.id,
+               Request:             "env",
+               WantReply:           true,
+               RequestSpecificData: payload,
+       })
+}
+
+// An empty mode list (a string of 1 character, opcode 0), see RFC 4254 Section 8.
+var emptyModeList = []byte{0, 0, 0, 1, 0}
+
+// RequestPty requests the association of a pty with the session on the remote host.
+func (s *Session) RequestPty(term string, h, w int) os.Error {
+       buf := make([]byte, 4+len(term)+16+len(emptyModeList))
+       b := marshalString(buf, []byte(term))
+       binary.BigEndian.PutUint32(b, uint32(h))
+       binary.BigEndian.PutUint32(b[4:], uint32(w))
+       binary.BigEndian.PutUint32(b[8:], uint32(h*8))
+       binary.BigEndian.PutUint32(b[12:], uint32(w*8))
+       copy(b[16:], emptyModeList)
+
+       return s.sendChanReq(channelRequestMsg{
+               PeersId:             s.id,
+               Request:             "pty-req",
+               WantReply:           true,
+               RequestSpecificData: buf,
+       })
+}
+
+// Exec runs cmd on the remote host. Typically, the remote 
+// server passes cmd to the shell for interpretation. 
+// A Session only accepts one call to Exec or Shell.
+func (s *Session) Exec(cmd string) os.Error {
+       if s.started {
+               return os.NewError("session already started")
+       }
+       cmdLen := stringLength([]byte(cmd))
+       payload := make([]byte, cmdLen)
+       marshalString(payload, []byte(cmd))
+       s.started = true
+
+       return s.sendChanReq(channelRequestMsg{
+               PeersId:             s.id,
+               Request:             "exec",
+               WantReply:           true,
+               RequestSpecificData: payload,
+       })
+}
+
+// Shell starts a login shell on the remote host. A Session only 
+// accepts one call to Exec or Shell.
+func (s *Session) Shell() os.Error {
+       if s.started {
+               return os.NewError("session already started")
+       }
+       s.started = true
+
+       return s.sendChanReq(channelRequestMsg{
+               PeersId:   s.id,
+               Request:   "shell",
+               WantReply: true,
+       })
+}
+
+// NewSession returns a new interactive session on the remote host.
+func (c *ClientConn) NewSession() (*Session, os.Error) {
+       ch, err := c.openChan("session")
+       if err != nil {
+               return nil, err
+       }
+       return &Session{
+               Stdin: &chanWriter{
+                       packetWriter: ch,
+                       id:           ch.id,
+                       win:          ch.win,
+               },
+               Stdout: &chanReader{
+                       packetWriter: ch,
+                       id:           ch.id,
+                       data:         ch.data,
+               },
+               Stderr: &chanReader{
+                       packetWriter: ch,
+                       id:           ch.id,
+                       data:         ch.dataExt,
+               },
+               clientChan: ch,
+       }, nil
+}
index 5994004..97eaf97 100644 (file)
@@ -332,16 +332,15 @@ func (t truncatingMAC) Size() int {
 const maxVersionStringBytes = 1024
 
 // Read version string as specified by RFC 4253, section 4.2.
-func readVersion(r io.Reader) (versionString []byte, ok bool) {
-       versionString = make([]byte, 0, 64)
-       seenCR := false
-
+func readVersion(r io.Reader) ([]byte, os.Error) {
+       versionString := make([]byte, 0, 64)
+       var ok, seenCR bool
        var buf [1]byte
 forEachByte:
        for len(versionString) < maxVersionStringBytes {
                _, err := io.ReadFull(r, buf[:])
                if err != nil {
-                       return
+                       return nil, err
                }
                b := buf[0]
 
@@ -360,10 +359,10 @@ forEachByte:
                versionString = append(versionString, b)
        }
 
-       if ok {
-               // We need to remove the CR from versionString
-               versionString = versionString[:len(versionString)-1]
+       if !ok {
+               return nil, os.NewError("failed to read version string")
        }
 
-       return
+       // We need to remove the CR from versionString
+       return versionString[:len(versionString)-1], nil
 }
index 9a610a7..b2e2a7f 100644 (file)
@@ -12,9 +12,9 @@ import (
 
 func TestReadVersion(t *testing.T) {
        buf := []byte(serverVersion)
-       result, ok := readVersion(bufio.NewReader(bytes.NewBuffer(buf)))
-       if !ok {
-               t.Error("readVersion didn't read version correctly")
+       result, err := readVersion(bufio.NewReader(bytes.NewBuffer(buf)))
+       if err != nil {
+               t.Errorf("readVersion didn't read version correctly: %s", err)
        }
        if !bytes.Equal(buf[:len(buf)-2], result) {
                t.Error("version read did not match expected")
@@ -23,7 +23,7 @@ func TestReadVersion(t *testing.T) {
 
 func TestReadVersionTooLong(t *testing.T) {
        buf := make([]byte, maxVersionStringBytes+1)
-       if _, ok := readVersion(bufio.NewReader(bytes.NewBuffer(buf))); ok {
+       if _, err := readVersion(bufio.NewReader(bytes.NewBuffer(buf))); err == nil {
                t.Errorf("readVersion consumed %d bytes without error", len(buf))
        }
 }
@@ -31,7 +31,7 @@ func TestReadVersionTooLong(t *testing.T) {
 func TestReadVersionWithoutCRLF(t *testing.T) {
        buf := []byte(serverVersion)
        buf = buf[:len(buf)-1]
-       if _, ok := readVersion(bufio.NewReader(bytes.NewBuffer(buf))); ok {
+       if _, err := readVersion(bufio.NewReader(bytes.NewBuffer(buf))); err == nil {
                t.Error("readVersion did not notice \\n was missing")
        }
 }
index fe90f91..e744a63 100644 (file)
@@ -289,9 +289,10 @@ func (p *gcParser) parseExportedName() (*ast.Object, string) {
 // BasicType = identifier .
 //
 func (p *gcParser) parseBasicType() Type {
-       obj := Universe.Lookup(p.expect(scanner.Ident))
+       id := p.expect(scanner.Ident)
+       obj := Universe.Lookup(id)
        if obj == nil || obj.Kind != ast.Typ {
-               p.errorf("not a basic type: %s", obj.Name)
+               p.errorf("not a basic type: %s", id)
        }
        return obj.Type.(Type)
 }
index edf2165..6e264d0 100644 (file)
@@ -6,8 +6,8 @@ package winfsnotify
 
 import (
        "os"
-       "time"
        "testing"
+       "time"
 )
 
 func expect(t *testing.T, eventstream <-chan *Event, name string, mask uint32) {
@@ -70,15 +70,11 @@ func TestNotifyEvents(t *testing.T) {
        if _, err = file.WriteString("hello, world"); err != nil {
                t.Fatalf("failed to write to test file: %s", err)
        }
-       if err = file.Sync(); err != nil {
-               t.Fatalf("failed to sync test file: %s", err)
-       }
-       expect(t, watcher.Event, testFile, FS_MODIFY)
-       expect(t, watcher.Event, testFile, FS_MODIFY)
-
        if err = file.Close(); err != nil {
                t.Fatalf("failed to close test file: %s", err)
        }
+       expect(t, watcher.Event, testFile, FS_MODIFY)
+       expect(t, watcher.Event, testFile, FS_MODIFY)
 
        if err = os.Rename(testFile, testFile2); err != nil {
                t.Fatalf("failed to rename test file: %s", err)
index 030ad61..38280d6 100644 (file)
@@ -88,6 +88,10 @@ type S struct {
        G G // a struct field that GoStrings
 }
 
+type SI struct {
+       I interface{}
+}
+
 // A type with a String method with pointer receiver for testing %p
 type P int
 
@@ -352,6 +356,7 @@ var fmttests = []struct {
        {"%#v", map[string]int{"a": 1}, `map[string] int{"a":1}`},
        {"%#v", map[string]B{"a": {1, 2}}, `map[string] fmt_test.B{"a":fmt_test.B{I:1, j:2}}`},
        {"%#v", []string{"a", "b"}, `[]string{"a", "b"}`},
+       {"%#v", SI{}, `fmt_test.SI{I:interface { }(nil)}`},
 
        // slices with other formats
        {"%#x", []int{1, 2, 15}, `[0x1 0x2 0xf]`},
index 7721e72..710baee 100644 (file)
@@ -74,6 +74,8 @@ type pp struct {
        n         int
        panicking bool
        buf       bytes.Buffer
+       // field holds the current item, as an interface{}.
+       field interface{}
        // value holds the current item, as a reflect.Value, and will be
        // the zero Value if the item has not been reflected.
        value   reflect.Value
@@ -132,6 +134,7 @@ func (p *pp) free() {
                return
        }
        p.buf.Reset()
+       p.field = nil
        p.value = reflect.Value{}
        ppFree.put(p)
 }
@@ -294,16 +297,16 @@ func (p *pp) unknownType(v interface{}) {
        p.buf.WriteByte('?')
 }
 
-func (p *pp) badVerb(verb int, val interface{}) {
+func (p *pp) badVerb(verb int) {
        p.add('%')
        p.add('!')
        p.add(verb)
        p.add('(')
        switch {
-       case val != nil:
-               p.buf.WriteString(reflect.TypeOf(val).String())
+       case p.field != nil:
+               p.buf.WriteString(reflect.TypeOf(p.field).String())
                p.add('=')
-               p.printField(val, 'v', false, false, 0)
+               p.printField(p.field, 'v', false, false, 0)
        case p.value.IsValid():
                p.buf.WriteString(p.value.Type().String())
                p.add('=')
@@ -314,12 +317,12 @@ func (p *pp) badVerb(verb int, val interface{}) {
        p.add(')')
 }
 
-func (p *pp) fmtBool(v bool, verb int, value interface{}) {
+func (p *pp) fmtBool(v bool, verb int) {
        switch verb {
        case 't', 'v':
                p.fmt.fmt_boolean(v)
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
@@ -333,7 +336,7 @@ func (p *pp) fmtC(c int64) {
        p.fmt.pad(p.runeBuf[0:w])
 }
 
-func (p *pp) fmtInt64(v int64, verb int, value interface{}) {
+func (p *pp) fmtInt64(v int64, verb int) {
        switch verb {
        case 'b':
                p.fmt.integer(v, 2, signed, ldigits)
@@ -347,7 +350,7 @@ func (p *pp) fmtInt64(v int64, verb int, value interface{}) {
                if 0 <= v && v <= unicode.MaxRune {
                        p.fmt.fmt_qc(v)
                } else {
-                       p.badVerb(verb, value)
+                       p.badVerb(verb)
                }
        case 'x':
                p.fmt.integer(v, 16, signed, ldigits)
@@ -356,7 +359,7 @@ func (p *pp) fmtInt64(v int64, verb int, value interface{}) {
        case 'X':
                p.fmt.integer(v, 16, signed, udigits)
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
@@ -391,7 +394,7 @@ func (p *pp) fmtUnicode(v int64) {
        p.fmt.sharp = sharp
 }
 
-func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) {
+func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool) {
        switch verb {
        case 'b':
                p.fmt.integer(int64(v), 2, unsigned, ldigits)
@@ -411,7 +414,7 @@ func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) {
                if 0 <= v && v <= unicode.MaxRune {
                        p.fmt.fmt_qc(int64(v))
                } else {
-                       p.badVerb(verb, value)
+                       p.badVerb(verb)
                }
        case 'x':
                p.fmt.integer(int64(v), 16, unsigned, ldigits)
@@ -420,11 +423,11 @@ func (p *pp) fmtUint64(v uint64, verb int, goSyntax bool, value interface{}) {
        case 'U':
                p.fmtUnicode(int64(v))
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
-func (p *pp) fmtFloat32(v float32, verb int, value interface{}) {
+func (p *pp) fmtFloat32(v float32, verb int) {
        switch verb {
        case 'b':
                p.fmt.fmt_fb32(v)
@@ -439,11 +442,11 @@ func (p *pp) fmtFloat32(v float32, verb int, value interface{}) {
        case 'G':
                p.fmt.fmt_G32(v)
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
-func (p *pp) fmtFloat64(v float64, verb int, value interface{}) {
+func (p *pp) fmtFloat64(v float64, verb int) {
        switch verb {
        case 'b':
                p.fmt.fmt_fb64(v)
@@ -458,33 +461,33 @@ func (p *pp) fmtFloat64(v float64, verb int, value interface{}) {
        case 'G':
                p.fmt.fmt_G64(v)
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
-func (p *pp) fmtComplex64(v complex64, verb int, value interface{}) {
+func (p *pp) fmtComplex64(v complex64, verb int) {
        switch verb {
        case 'e', 'E', 'f', 'F', 'g', 'G':
                p.fmt.fmt_c64(v, verb)
        case 'v':
                p.fmt.fmt_c64(v, 'g')
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
-func (p *pp) fmtComplex128(v complex128, verb int, value interface{}) {
+func (p *pp) fmtComplex128(v complex128, verb int) {
        switch verb {
        case 'e', 'E', 'f', 'F', 'g', 'G':
                p.fmt.fmt_c128(v, verb)
        case 'v':
                p.fmt.fmt_c128(v, 'g')
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
-func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}) {
+func (p *pp) fmtString(v string, verb int, goSyntax bool) {
        switch verb {
        case 'v':
                if goSyntax {
@@ -501,11 +504,11 @@ func (p *pp) fmtString(v string, verb int, goSyntax bool, value interface{}) {
        case 'q':
                p.fmt.fmt_q(v)
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
-func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interface{}) {
+func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int) {
        if verb == 'v' || verb == 'd' {
                if goSyntax {
                        p.buf.Write(bytesBytes)
@@ -540,17 +543,17 @@ func (p *pp) fmtBytes(v []byte, verb int, goSyntax bool, depth int, value interf
        case 'q':
                p.fmt.fmt_q(s)
        default:
-               p.badVerb(verb, value)
+               p.badVerb(verb)
        }
 }
 
-func (p *pp) fmtPointer(field interface{}, value reflect.Value, verb int, goSyntax bool) {
+func (p *pp) fmtPointer(value reflect.Value, verb int, goSyntax bool) {
        var u uintptr
        switch value.Kind() {
        case reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer:
                u = value.Pointer()
        default:
-               p.badVerb(verb, field)
+               p.badVerb(verb)
                return
        }
        if goSyntax {
@@ -576,12 +579,12 @@ var (
        uintptrBits = reflect.TypeOf(uintptr(0)).Bits()
 )
 
-func (p *pp) catchPanic(val interface{}, verb int) {
+func (p *pp) catchPanic(field interface{}, verb int) {
        if err := recover(); err != nil {
                // If it's a nil pointer, just say "<nil>". The likeliest causes are a
                // Stringer that fails to guard against nil or a nil pointer for a
                // value receiver, and in either case, "<nil>" is a nice result.
-               if v := reflect.ValueOf(val); v.Kind() == reflect.Ptr && v.IsNil() {
+               if v := reflect.ValueOf(field); v.Kind() == reflect.Ptr && v.IsNil() {
                        p.buf.Write(nilAngleBytes)
                        return
                }
@@ -601,12 +604,12 @@ func (p *pp) catchPanic(val interface{}, verb int) {
        }
 }
 
-func (p *pp) handleMethods(field interface{}, verb int, plus, goSyntax bool, depth int) (wasString, handled bool) {
+func (p *pp) handleMethods(verb int, plus, goSyntax bool, depth int) (wasString, handled bool) {
        // Is it a Formatter?
-       if formatter, ok := field.(Formatter); ok {
+       if formatter, ok := p.field.(Formatter); ok {
                handled = true
                wasString = false
-               defer p.catchPanic(field, verb)
+               defer p.catchPanic(p.field, verb)
                formatter.Format(p, verb)
                return
        }
@@ -618,20 +621,20 @@ func (p *pp) handleMethods(field interface{}, verb int, plus, goSyntax bool, dep
        // If we're doing Go syntax and the field knows how to supply it, take care of it now.
        if goSyntax {
                p.fmt.sharp = false
-               if stringer, ok := field.(GoStringer); ok {
+               if stringer, ok := p.field.(GoStringer); ok {
                        wasString = false
                        handled = true
-                       defer p.catchPanic(field, verb)
+                       defer p.catchPanic(p.field, verb)
                        // Print the result of GoString unadorned.
-                       p.fmtString(stringer.GoString(), 's', false, field)
+                       p.fmtString(stringer.GoString(), 's', false)
                        return
                }
        } else {
                // Is it a Stringer?
-               if stringer, ok := field.(Stringer); ok {
+               if stringer, ok := p.field.(Stringer); ok {
                        wasString = false
                        handled = true
-                       defer p.catchPanic(field, verb)
+                       defer p.catchPanic(p.field, verb)
                        p.printField(stringer.String(), verb, plus, false, depth)
                        return
                }
@@ -645,11 +648,13 @@ func (p *pp) printField(field interface{}, verb int, plus, goSyntax bool, depth
                if verb == 'T' || verb == 'v' {
                        p.buf.Write(nilAngleBytes)
                } else {
-                       p.badVerb(verb, field)
+                       p.badVerb(verb)
                }
                return false
        }
 
+       p.field = field
+       p.value = reflect.Value{}
        // Special processing considerations.
        // %T (the value's type) and %p (its address) are special; we always do them first.
        switch verb {
@@ -657,74 +662,60 @@ func (p *pp) printField(field interface{}, verb int, plus, goSyntax bool, depth
                p.printField(reflect.TypeOf(field).String(), 's', false, false, 0)
                return false
        case 'p':
-               p.fmtPointer(field, reflect.ValueOf(field), verb, goSyntax)
+               p.fmtPointer(reflect.ValueOf(field), verb, goSyntax)
                return false
        }
 
-       if wasString, handled := p.handleMethods(field, verb, plus, goSyntax, depth); handled {
+       if wasString, handled := p.handleMethods(verb, plus, goSyntax, depth); handled {
                return wasString
        }
 
        // Some types can be done without reflection.
        switch f := field.(type) {
        case bool:
-               p.fmtBool(f, verb, field)
-               return false
+               p.fmtBool(f, verb)
        case float32:
-               p.fmtFloat32(f, verb, field)
-               return false
+               p.fmtFloat32(f, verb)
        case float64:
-               p.fmtFloat64(f, verb, field)
-               return false
+               p.fmtFloat64(f, verb)
        case complex64:
-               p.fmtComplex64(complex64(f), verb, field)
-               return false
+               p.fmtComplex64(complex64(f), verb)
        case complex128:
-               p.fmtComplex128(f, verb, field)
-               return false
+               p.fmtComplex128(f, verb)
        case int:
-               p.fmtInt64(int64(f), verb, field)
-               return false
+               p.fmtInt64(int64(f), verb)
        case int8:
-               p.fmtInt64(int64(f), verb, field)
-               return false
+               p.fmtInt64(int64(f), verb)
        case int16:
-               p.fmtInt64(int64(f), verb, field)
-               return false
+               p.fmtInt64(int64(f), verb)
        case int32:
-               p.fmtInt64(int64(f), verb, field)
-               return false
+               p.fmtInt64(int64(f), verb)
        case int64:
-               p.fmtInt64(f, verb, field)
-               return false
+               p.fmtInt64(f, verb)
        case uint:
-               p.fmtUint64(uint64(f), verb, goSyntax, field)
-               return false
+               p.fmtUint64(uint64(f), verb, goSyntax)
        case uint8:
-               p.fmtUint64(uint64(f), verb, goSyntax, field)
-               return false
+               p.fmtUint64(uint64(f), verb, goSyntax)
        case uint16:
-               p.fmtUint64(uint64(f), verb, goSyntax, field)
-               return false
+               p.fmtUint64(uint64(f), verb, goSyntax)
        case uint32:
-               p.fmtUint64(uint64(f), verb, goSyntax, field)
-               return false
+               p.fmtUint64(uint64(f), verb, goSyntax)
        case uint64:
-               p.fmtUint64(f, verb, goSyntax, field)
-               return false
+               p.fmtUint64(f, verb, goSyntax)
        case uintptr:
-               p.fmtUint64(uint64(f), verb, goSyntax, field)
-               return false
+               p.fmtUint64(uint64(f), verb, goSyntax)
        case string:
-               p.fmtString(f, verb, goSyntax, field)
-               return verb == 's' || verb == 'v'
+               p.fmtString(f, verb, goSyntax)
+               wasString = verb == 's' || verb == 'v'
        case []byte:
-               p.fmtBytes(f, verb, goSyntax, depth, field)
-               return verb == 's'
+               p.fmtBytes(f, verb, goSyntax, depth)
+               wasString = verb == 's'
+       default:
+               // Need to use reflection
+               return p.printReflectValue(reflect.ValueOf(field), verb, plus, goSyntax, depth)
        }
-
-       // Need to use reflection
-       return p.printReflectValue(reflect.ValueOf(field), verb, plus, goSyntax, depth)
+       p.field = nil
+       return
 }
 
 // printValue is like printField but starts with a reflect value, not an interface{} value.
@@ -733,7 +724,7 @@ func (p *pp) printValue(value reflect.Value, verb int, plus, goSyntax bool, dept
                if verb == 'T' || verb == 'v' {
                        p.buf.Write(nilAngleBytes)
                } else {
-                       p.badVerb(verb, nil)
+                       p.badVerb(verb)
                }
                return false
        }
@@ -745,17 +736,17 @@ func (p *pp) printValue(value reflect.Value, verb int, plus, goSyntax bool, dept
                p.printField(value.Type().String(), 's', false, false, 0)
                return false
        case 'p':
-               p.fmtPointer(nil, value, verb, goSyntax)
+               p.fmtPointer(value, verb, goSyntax)
                return false
        }
 
        // Handle values with special methods.
        // Call always, even when field == nil, because handleMethods clears p.fmt.plus for us.
-       var field interface{}
+       p.field = nil // Make sure it's cleared, for safety.
        if value.CanInterface() {
-               field = value.Interface()
+               p.field = value.Interface()
        }
-       if wasString, handled := p.handleMethods(field, verb, plus, goSyntax, depth); handled {
+       if wasString, handled := p.handleMethods(verb, plus, goSyntax, depth); handled {
                return wasString
        }
 
@@ -770,25 +761,25 @@ func (p *pp) printReflectValue(value reflect.Value, verb int, plus, goSyntax boo
 BigSwitch:
        switch f := value; f.Kind() {
        case reflect.Bool:
-               p.fmtBool(f.Bool(), verb, nil)
+               p.fmtBool(f.Bool(), verb)
        case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-               p.fmtInt64(f.Int(), verb, nil)
+               p.fmtInt64(f.Int(), verb)
        case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
-               p.fmtUint64(uint64(f.Uint()), verb, goSyntax, nil)
+               p.fmtUint64(uint64(f.Uint()), verb, goSyntax)
        case reflect.Float32, reflect.Float64:
                if f.Type().Size() == 4 {
-                       p.fmtFloat32(float32(f.Float()), verb, nil)
+                       p.fmtFloat32(float32(f.Float()), verb)
                } else {
-                       p.fmtFloat64(float64(f.Float()), verb, nil)
+                       p.fmtFloat64(float64(f.Float()), verb)
                }
        case reflect.Complex64, reflect.Complex128:
                if f.Type().Size() == 8 {
-                       p.fmtComplex64(complex64(f.Complex()), verb, nil)
+                       p.fmtComplex64(complex64(f.Complex()), verb)
                } else {
-                       p.fmtComplex128(complex128(f.Complex()), verb, nil)
+                       p.fmtComplex128(complex128(f.Complex()), verb)
                }
        case reflect.String:
-               p.fmtString(f.String(), verb, goSyntax, nil)
+               p.fmtString(f.String(), verb, goSyntax)
        case reflect.Map:
                if goSyntax {
                        p.buf.WriteString(f.Type().String())
@@ -842,7 +833,7 @@ BigSwitch:
                value := f.Elem()
                if !value.IsValid() {
                        if goSyntax {
-                               p.buf.WriteString(value.Type().String())
+                               p.buf.WriteString(f.Type().String())
                                p.buf.Write(nilParenBytes)
                        } else {
                                p.buf.Write(nilAngleBytes)
@@ -864,7 +855,7 @@ BigSwitch:
                        for i := range bytes {
                                bytes[i] = byte(f.Index(i).Uint())
                        }
-                       p.fmtBytes(bytes, verb, goSyntax, depth, nil)
+                       p.fmtBytes(bytes, verb, goSyntax, depth)
                        wasString = verb == 's'
                        break
                }
@@ -924,7 +915,7 @@ BigSwitch:
                }
                p.fmt0x64(uint64(v), true)
        case reflect.Chan, reflect.Func, reflect.UnsafePointer:
-               p.fmtPointer(nil, value, verb, goSyntax)
+               p.fmtPointer(value, verb, goSyntax)
        default:
                p.unknownType(f)
        }
index a4bc3bb..c3153ed 100644 (file)
@@ -23,11 +23,10 @@ var tests = []struct {
        {"foobar", "0  \"foobar\""},
 
        // maps
-       {map[string]int{"a": 1, "b": 2},
-               `0  map[string] int (len = 2) {
+       {map[string]int{"a": 1},
+               `0  map[string] int (len = 1) {
                1  .  "a": 1
-               2  .  "b": 2
-               3  }`},
+               2  }`},
 
        // pointers
        {new(int), "0  *0"},
index 5bc0630..ba9d188 100644 (file)
@@ -70,9 +70,6 @@ call to Next. For example, to extract an HTML page's anchor text:
                }
        }
 
-A Tokenizer typically skips over HTML comments. To return comment tokens, set
-Tokenizer.ReturnComments to true before looping over calls to Next.
-
 Parsing is done by calling Parse with an io.Reader, which returns the root of
 the parse tree (the document element) as a *Node. It is the caller's
 responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
index 582437f..530942a 100644 (file)
@@ -32,6 +32,9 @@ type parser struct {
        // originalIM is the insertion mode to go back to after completing a text
        // or inTableText insertion mode.
        originalIM insertionMode
+       // fosterParenting is whether new elements should be inserted according to
+       // the foster parenting rules (section 11.2.5.3).
+       fosterParenting bool
 }
 
 func (p *parser) top() *Node {
@@ -49,6 +52,11 @@ var (
        tableScopeStopTags    = []string{"html", "table"}
 )
 
+// stopTags for use in clearStackToContext.
+var (
+       tableRowContextStopTags = []string{"tr", "html"}
+)
+
 // popUntil pops the stack of open elements at the highest element whose tag
 // is in matchTags, provided there is no higher element in stopTags. It returns
 // whether or not there was such an element. If there was not, popUntil leaves
@@ -103,12 +111,61 @@ func (p *parser) elementInScope(stopTags []string, matchTags ...string) bool {
 // addChild adds a child node n to the top element, and pushes n onto the stack
 // of open elements if it is an element node.
 func (p *parser) addChild(n *Node) {
-       p.top().Add(n)
+       if p.fosterParenting {
+               p.fosterParent(n)
+       } else {
+               p.top().Add(n)
+       }
+
        if n.Type == ElementNode {
                p.oe = append(p.oe, n)
        }
 }
 
+// fosterParent adds a child node according to the foster parenting rules.
+// Section 11.2.5.3, "foster parenting".
+func (p *parser) fosterParent(n *Node) {
+       var table, parent *Node
+       var i int
+       for i = len(p.oe) - 1; i >= 0; i-- {
+               if p.oe[i].Data == "table" {
+                       table = p.oe[i]
+                       break
+               }
+       }
+
+       if table == nil {
+               // The foster parent is the html element.
+               parent = p.oe[0]
+       } else {
+               parent = table.Parent
+       }
+       if parent == nil {
+               parent = p.oe[i-1]
+       }
+
+       var child *Node
+       for i, child = range parent.Child {
+               if child == table {
+                       break
+               }
+       }
+
+       if i > 0 && parent.Child[i-1].Type == TextNode && n.Type == TextNode {
+               parent.Child[i-1].Data += n.Data
+               return
+       }
+
+       if i == len(parent.Child) {
+               parent.Add(n)
+       } else {
+               // Insert n into parent.Child at index i.
+               parent.Child = append(parent.Child[:i+1], parent.Child[i:]...)
+               parent.Child[i] = n
+               n.Parent = parent
+       }
+}
+
 // addText adds text to the preceding node if it is a text node, or else it
 // calls addChild with a new text node.
 func (p *parser) addText(text string) {
@@ -170,9 +227,9 @@ func (p *parser) reconstructActiveFormattingElements() {
        }
        for {
                i++
-               n = p.afe[i]
-               p.addChild(n.clone())
-               p.afe[i] = n
+               clone := p.afe[i].clone()
+               p.addChild(clone)
+               p.afe[i] = clone
                if i == len(p.afe)-1 {
                        break
                }
@@ -234,10 +291,52 @@ func (p *parser) setOriginalIM(im insertionMode) {
        p.originalIM = im
 }
 
+// Section 11.2.3.1, "reset the insertion mode".
+func (p *parser) resetInsertionMode() insertionMode {
+       for i := len(p.oe) - 1; i >= 0; i-- {
+               n := p.oe[i]
+               if i == 0 {
+                       // TODO: set n to the context element, for HTML fragment parsing.
+               }
+               switch n.Data {
+               case "select":
+                       return inSelectIM
+               case "td", "th":
+                       return inCellIM
+               case "tr":
+                       return inRowIM
+               case "tbody", "thead", "tfoot":
+                       return inTableBodyIM
+               case "caption":
+                       // TODO: return inCaptionIM
+               case "colgroup":
+                       // TODO: return inColumnGroupIM
+               case "table":
+                       return inTableIM
+               case "head":
+                       return inBodyIM
+               case "body":
+                       return inBodyIM
+               case "frameset":
+                       // TODO: return inFramesetIM
+               case "html":
+                       return beforeHeadIM
+               }
+       }
+       return inBodyIM
+}
+
 // Section 11.2.5.4.1.
 func initialIM(p *parser) (insertionMode, bool) {
-       if p.tok.Type == DoctypeToken {
-               p.addChild(&Node{
+       switch p.tok.Type {
+       case CommentToken:
+               p.doc.Add(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return initialIM, true
+       case DoctypeToken:
+               p.doc.Add(&Node{
                        Type: DoctypeNode,
                        Data: p.tok.Data,
                })
@@ -275,6 +374,12 @@ func beforeHTMLIM(p *parser) (insertionMode, bool) {
                default:
                        // Ignore the token.
                }
+       case CommentToken:
+               p.doc.Add(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return beforeHTMLIM, true
        }
        if add || implied {
                p.addElement("html", attr)
@@ -312,6 +417,12 @@ func beforeHeadIM(p *parser) (insertionMode, bool) {
                default:
                        // Ignore the token.
                }
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return beforeHeadIM, true
        }
        if add || implied {
                p.addElement("head", attr)
@@ -344,11 +455,17 @@ func inHeadIM(p *parser) (insertionMode, bool) {
                        pop = true
                }
                // TODO.
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return inHeadIM, true
        }
        if pop || implied {
                n := p.oe.pop()
                if n.Data != "head" {
-                       panic("html: bad parser state")
+                       panic("html: bad parser state: <head> element not found, in the in-head insertion mode")
                }
                return afterHeadIM, !implied
        }
@@ -387,6 +504,12 @@ func afterHeadIM(p *parser) (insertionMode, bool) {
                }
        case EndTagToken:
                // TODO.
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return afterHeadIM, true
        }
        if add || implied {
                p.addElement("body", attr)
@@ -447,6 +570,30 @@ func inBodyIM(p *parser) (insertionMode, bool) {
                        p.oe.pop()
                        p.acknowledgeSelfClosingTag()
                        p.framesetOK = false
+               case "select":
+                       p.reconstructActiveFormattingElements()
+                       p.addElement(p.tok.Data, p.tok.Attr)
+                       p.framesetOK = false
+                       // TODO: detect <select> inside a table.
+                       return inSelectIM, true
+               case "li":
+                       p.framesetOK = false
+                       for i := len(p.oe) - 1; i >= 0; i-- {
+                               node := p.oe[i]
+                               switch node.Data {
+                               case "li":
+                                       p.popUntil(listItemScopeStopTags, "li")
+                               case "address", "div", "p":
+                                       continue
+                               default:
+                                       if !isSpecialElement[node.Data] {
+                                               continue
+                                       }
+                               }
+                               break
+                       }
+                       p.popUntil(buttonScopeStopTags, "p")
+                       p.addElement("li", p.tok.Attr)
                default:
                        // TODO.
                        p.addElement(p.tok.Data, p.tok.Attr)
@@ -463,12 +610,16 @@ func inBodyIM(p *parser) (insertionMode, bool) {
                        p.popUntil(buttonScopeStopTags, "p")
                case "a", "b", "big", "code", "em", "font", "i", "nobr", "s", "small", "strike", "strong", "tt", "u":
                        p.inBodyEndTagFormatting(p.tok.Data)
+               case "address", "article", "aside", "blockquote", "button", "center", "details", "dir", "div", "dl", "fieldset", "figcaption", "figure", "footer", "header", "hgroup", "listing", "menu", "nav", "ol", "pre", "section", "summary", "ul":
+                       p.popUntil(defaultScopeStopTags, p.tok.Data)
                default:
-                       // TODO: any other end tag
-                       if p.tok.Data == p.top().Data {
-                               p.oe.pop()
-                       }
+                       p.inBodyEndTagOther(p.tok.Data)
                }
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
        }
 
        return inBodyIM, true
@@ -496,6 +647,7 @@ func (p *parser) inBodyEndTagFormatting(tag string) {
                        }
                }
                if formattingElement == nil {
+                       p.inBodyEndTagOther(tag)
                        return
                }
                feIndex := p.oe.index(formattingElement)
@@ -568,8 +720,7 @@ func (p *parser) inBodyEndTagFormatting(tag string) {
                }
                switch commonAncestor.Data {
                case "table", "tbody", "tfoot", "thead", "tr":
-                       // TODO: fix up misnested table nodes; find the foster parent.
-                       fallthrough
+                       p.fosterParent(lastNode)
                default:
                        commonAncestor.Add(lastNode)
                }
@@ -590,6 +741,19 @@ func (p *parser) inBodyEndTagFormatting(tag string) {
        }
 }
 
+// inBodyEndTagOther performs the "any other end tag" algorithm for inBodyIM.
+func (p *parser) inBodyEndTagOther(tag string) {
+       for i := len(p.oe) - 1; i >= 0; i-- {
+               if p.oe[i].Data == tag {
+                       p.oe = p.oe[:i]
+                       break
+               }
+               if isSpecialElement[p.oe[i].Data] {
+                       break
+               }
+       }
+}
+
 // Section 11.2.5.4.8.
 func textIM(p *parser) (insertionMode, bool) {
        switch p.tok.Type {
@@ -606,12 +770,6 @@ func textIM(p *parser) (insertionMode, bool) {
 
 // Section 11.2.5.4.9.
 func inTableIM(p *parser) (insertionMode, bool) {
-       var (
-               add      bool
-               data     string
-               attr     []Attribute
-               consumed bool
-       )
        switch p.tok.Type {
        case ErrorToken:
                // Stop parsing.
@@ -621,13 +779,19 @@ func inTableIM(p *parser) (insertionMode, bool) {
        case StartTagToken:
                switch p.tok.Data {
                case "tbody", "tfoot", "thead":
-                       add = true
-                       data = p.tok.Data
-                       attr = p.tok.Attr
-                       consumed = true
+                       p.clearStackToContext(tableScopeStopTags)
+                       p.addElement(p.tok.Data, p.tok.Attr)
+                       return inTableBodyIM, true
                case "td", "th", "tr":
-                       add = true
-                       data = "tbody"
+                       p.clearStackToContext(tableScopeStopTags)
+                       p.addElement("tbody", nil)
+                       return inTableBodyIM, false
+               case "table":
+                       if p.popUntil(tableScopeStopTags, "table") {
+                               return p.resetInsertionMode(), false
+                       }
+                       // Ignore the token.
+                       return inTableIM, true
                default:
                        // TODO.
                }
@@ -635,8 +799,7 @@ func inTableIM(p *parser) (insertionMode, bool) {
                switch p.tok.Data {
                case "table":
                        if p.popUntil(tableScopeStopTags, "table") {
-                               // TODO: "reset the insertion mode appropriately" as per 11.2.3.1.
-                               return inBodyIM, false
+                               return p.resetInsertionMode(), true
                        }
                        // Ignore the token.
                        return inTableIM, true
@@ -644,14 +807,34 @@ func inTableIM(p *parser) (insertionMode, bool) {
                        // Ignore the token.
                        return inTableIM, true
                }
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return inTableIM, true
        }
-       if add {
-               // TODO: clear the stack back to a table context.
-               p.addElement(data, attr)
-               return inTableBodyIM, consumed
+
+       switch p.top().Data {
+       case "table", "tbody", "tfoot", "thead", "tr":
+               p.fosterParenting = true
+               defer func() { p.fosterParenting = false }()
+       }
+
+       return useTheRulesFor(p, inTableIM, inBodyIM)
+}
+
+// clearStackToContext pops elements off the stack of open elements
+// until an element listed in stopTags is found.
+func (p *parser) clearStackToContext(stopTags []string) {
+       for i := len(p.oe) - 1; i >= 0; i-- {
+               for _, tag := range stopTags {
+                       if p.oe[i].Data == tag {
+                               p.oe = p.oe[:i+1]
+                               return
+                       }
+               }
        }
-       // TODO: return useTheRulesFor(inTableIM, inBodyIM, p) unless etc. etc. foster parenting.
-       return inTableIM, true
 }
 
 // Section 11.2.5.4.13.
@@ -693,6 +876,12 @@ func inTableBodyIM(p *parser) (insertionMode, bool) {
                        // Ignore the token.
                        return inTableBodyIM, true
                }
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return inTableBodyIM, true
        }
        if add {
                // TODO: clear the stack back to a table body context.
@@ -722,7 +911,12 @@ func inRowIM(p *parser) (insertionMode, bool) {
        case EndTagToken:
                switch p.tok.Data {
                case "tr":
-                       // TODO.
+                       if !p.elementInScope(tableScopeStopTags, "tr") {
+                               return inRowIM, true
+                       }
+                       p.clearStackToContext(tableRowContextStopTags)
+                       p.oe.pop()
+                       return inTableBodyIM, true
                case "table":
                        if p.popUntil(tableScopeStopTags, "tr") {
                                return inTableBodyIM, false
@@ -737,6 +931,12 @@ func inRowIM(p *parser) (insertionMode, bool) {
                default:
                        // TODO.
                }
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return inRowIM, true
        }
        return useTheRulesFor(p, inRowIM, inTableIM)
 }
@@ -763,6 +963,12 @@ func inCellIM(p *parser) (insertionMode, bool) {
                        // TODO: check for matching element in table scope.
                        closeTheCellAndReprocess = true
                }
+       case CommentToken:
+               p.addChild(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return inCellIM, true
        }
        if closeTheCellAndReprocess {
                if p.popUntil(tableScopeStopTags, "td") || p.popUntil(tableScopeStopTags, "th") {
@@ -773,6 +979,68 @@ func inCellIM(p *parser) (insertionMode, bool) {
        return useTheRulesFor(p, inCellIM, inBodyIM)
 }
 
+// Section 11.2.5.4.16.
+func inSelectIM(p *parser) (insertionMode, bool) {
+       endSelect := false
+       switch p.tok.Type {
+       case ErrorToken:
+               // TODO.
+       case TextToken:
+               p.addText(p.tok.Data)
+       case StartTagToken:
+               switch p.tok.Data {
+               case "html":
+                       // TODO.
+               case "option":
+                       if p.top().Data == "option" {
+                               p.oe.pop()
+                       }
+                       p.addElement(p.tok.Data, p.tok.Attr)
+               case "optgroup":
+                       // TODO.
+               case "select":
+                       endSelect = true
+               case "input", "keygen", "textarea":
+                       // TODO.
+               case "script":
+                       // TODO.
+               default:
+                       // Ignore the token.
+               }
+       case EndTagToken:
+               switch p.tok.Data {
+               case "option":
+                       // TODO.
+               case "optgroup":
+                       // TODO.
+               case "select":
+                       endSelect = true
+               default:
+                       // Ignore the token.
+               }
+       case CommentToken:
+               p.doc.Add(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+       }
+       if endSelect {
+               for i := len(p.oe) - 1; i >= 0; i-- {
+                       switch p.oe[i].Data {
+                       case "select":
+                               p.oe = p.oe[:i]
+                               return p.resetInsertionMode(), true
+                       case "option", "optgroup":
+                               continue
+                       default:
+                               // Ignore the token.
+                               return inSelectIM, true
+                       }
+               }
+       }
+       return inSelectIM, true
+}
+
 // Section 11.2.5.4.18.
 func afterBodyIM(p *parser) (insertionMode, bool) {
        switch p.tok.Type {
@@ -790,7 +1058,18 @@ func afterBodyIM(p *parser) (insertionMode, bool) {
                default:
                        // TODO.
                }
+       case CommentToken:
+               // The comment is attached to the <html> element.
+               if len(p.oe) < 1 || p.oe[0].Data != "html" {
+                       panic("html: bad parser state: <html> element not found, in the after-body insertion mode")
+               }
+               p.oe[0].Add(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return afterBodyIM, true
        }
+       // TODO: should this be "return inBodyIM, true"?
        return afterBodyIM, true
 }
 
@@ -806,6 +1085,12 @@ func afterAfterBodyIM(p *parser) (insertionMode, bool) {
                if p.tok.Data == "html" {
                        return useTheRulesFor(p, afterAfterBodyIM, inBodyIM)
                }
+       case CommentToken:
+               p.doc.Add(&Node{
+                       Type: CommentNode,
+                       Data: p.tok.Data,
+               })
+               return afterAfterBodyIM, true
        }
        return inBodyIM, false
 }
index 564580c..b0ddd92 100644 (file)
@@ -69,11 +69,15 @@ func readDat(filename string, c chan io.Reader) {
        }
 }
 
-func dumpLevel(w io.Writer, n *Node, level int) os.Error {
+func dumpIndent(w io.Writer, level int) {
        io.WriteString(w, "| ")
        for i := 0; i < level; i++ {
                io.WriteString(w, "  ")
        }
+}
+
+func dumpLevel(w io.Writer, n *Node, level int) os.Error {
+       dumpIndent(w, level)
        switch n.Type {
        case ErrorNode:
                return os.NewError("unexpected ErrorNode")
@@ -81,10 +85,15 @@ func dumpLevel(w io.Writer, n *Node, level int) os.Error {
                return os.NewError("unexpected DocumentNode")
        case ElementNode:
                fmt.Fprintf(w, "<%s>", n.Data)
+               for _, a := range n.Attr {
+                       io.WriteString(w, "\n")
+                       dumpIndent(w, level+1)
+                       fmt.Fprintf(w, `%s="%s"`, a.Key, a.Val)
+               }
        case TextNode:
                fmt.Fprintf(w, "%q", n.Data)
        case CommentNode:
-               return os.NewError("COMMENT")
+               fmt.Fprintf(w, "<!-- %s -->", n.Data)
        case DoctypeNode:
                fmt.Fprintf(w, "<!DOCTYPE %s>", n.Data)
        case scopeMarkerNode:
@@ -123,7 +132,7 @@ func TestParser(t *testing.T) {
                rc := make(chan io.Reader)
                go readDat(filename, rc)
                // TODO(nigeltao): Process all test cases, not just a subset.
-               for i := 0; i < 27; i++ {
+               for i := 0; i < 34; i++ {
                        // Parse the #data section.
                        b, err := ioutil.ReadAll(<-rc)
                        if err != nil {
@@ -152,6 +161,13 @@ func TestParser(t *testing.T) {
                                continue
                        }
                        // Check that rendering and re-parsing results in an identical tree.
+                       if filename == "tests1.dat" && i == 30 {
+                               // Test 30 in tests1.dat is such messed-up markup that a correct parse
+                               // results in a non-conforming tree (one <a> element nested inside another).
+                               // Therefore when it is rendered and re-parsed, it isn't the same.
+                               // So we skip rendering on that test.
+                               continue
+                       }
                        pr, pw := io.Pipe()
                        go func() {
                                pw.CloseWithError(Render(pw, doc))
index e1ec66f..d5dc448 100644 (file)
@@ -30,9 +30,6 @@ type writer interface {
 // would become a tree containing <html>, <head> and <body> elements. Another
 // example is that the programmatic equivalent of "a<head>b</head>c" becomes
 // "<html><head><head/><body>abc</body></html>".
-//
-// Comment nodes are elided from the output, analogous to Parse skipping over
-// any <!--comment--> input.
 func Render(w io.Writer, n *Node) os.Error {
        if x, ok := w.(writer); ok {
                return render(x, n)
@@ -61,6 +58,15 @@ func render(w writer, n *Node) os.Error {
        case ElementNode:
                // No-op.
        case CommentNode:
+               if _, err := w.WriteString("<!--"); err != nil {
+                       return err
+               }
+               if _, err := w.WriteString(n.Data); err != nil {
+                       return err
+               }
+               if _, err := w.WriteString("-->"); err != nil {
+                       return err
+               }
                return nil
        case DoctypeNode:
                if _, err := w.WriteString("<!DOCTYPE "); err != nil {
index 2826f95..952d174 100644 (file)
@@ -116,10 +116,6 @@ type span struct {
 
 // A Tokenizer returns a stream of HTML Tokens.
 type Tokenizer struct {
-       // If ReturnComments is set, Next returns comment tokens;
-       // otherwise it skips over comments (default).
-       ReturnComments bool
-
        // r is the source of the HTML text.
        r io.Reader
        // tt is the TokenType of the current token.
@@ -546,17 +542,19 @@ func (z *Tokenizer) readTagAttrVal() {
        }
 }
 
-// next scans the next token and returns its type.
-func (z *Tokenizer) next() TokenType {
+// Next scans the next token and returns its type.
+func (z *Tokenizer) Next() TokenType {
        if z.err != nil {
-               return ErrorToken
+               z.tt = ErrorToken
+               return z.tt
        }
        z.raw.start = z.raw.end
        z.data.start = z.raw.end
        z.data.end = z.raw.end
        if z.rawTag != "" {
                z.readRawOrRCDATA()
-               return TextToken
+               z.tt = TextToken
+               return z.tt
        }
        z.textIsRaw = false
 
@@ -596,11 +594,13 @@ loop:
                if x := z.raw.end - len("<a"); z.raw.start < x {
                        z.raw.end = x
                        z.data.end = x
-                       return TextToken
+                       z.tt = TextToken
+                       return z.tt
                }
                switch tokenType {
                case StartTagToken:
-                       return z.readStartTag()
+                       z.tt = z.readStartTag()
+                       return z.tt
                case EndTagToken:
                        c = z.readByte()
                        if z.err != nil {
@@ -616,39 +616,31 @@ loop:
                        }
                        if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
                                z.readEndTag()
-                               return EndTagToken
+                               z.tt = EndTagToken
+                               return z.tt
                        }
                        z.raw.end--
                        z.readUntilCloseAngle()
-                       return CommentToken
+                       z.tt = CommentToken
+                       return z.tt
                case CommentToken:
                        if c == '!' {
-                               return z.readMarkupDeclaration()
+                               z.tt = z.readMarkupDeclaration()
+                               return z.tt
                        }
                        z.raw.end--
                        z.readUntilCloseAngle()
-                       return CommentToken
+                       z.tt = CommentToken
+                       return z.tt
                }
        }
        if z.raw.start < z.raw.end {
                z.data.end = z.raw.end
-               return TextToken
-       }
-       return ErrorToken
-}
-
-// Next scans the next token and returns its type.
-func (z *Tokenizer) Next() TokenType {
-       for {
-               z.tt = z.next()
-               // TODO: remove the ReturnComments option. A tokenizer should
-               // always return comment tags.
-               if z.tt == CommentToken && !z.ReturnComments {
-                       continue
-               }
+               z.tt = TextToken
                return z.tt
        }
-       panic("unreachable")
+       z.tt = ErrorToken
+       return z.tt
 }
 
 // Raw returns the unmodified text of the current token. Calling Next, Token,
index 310cd97..45ce85e 100644 (file)
@@ -424,7 +424,6 @@ func TestTokenizer(t *testing.T) {
 loop:
        for _, tt := range tokenTests {
                z := NewTokenizer(strings.NewReader(tt.html))
-               z.ReturnComments = true
                if tt.golden != "" {
                        for i, s := range strings.Split(tt.golden, "$") {
                                if z.Next() == ErrorToken {
index 3fa4a05..e939b96 100644 (file)
@@ -2,7 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Primitive HTTP client. See RFC 2616.
+// HTTP client. See RFC 2616.
+// 
+// This is the high-level Client interface.
+// The low-level implementation is in transport.go.
 
 package http
 
index 0ad6cd7..8f61286 100644 (file)
@@ -7,6 +7,7 @@
 package http_test
 
 import (
+       "crypto/tls"
        "fmt"
        . "http"
        "http/httptest"
@@ -292,3 +293,26 @@ func TestClientWrites(t *testing.T) {
                t.Errorf("Post request did %d Write calls, want 1", writes)
        }
 }
+
+func TestClientInsecureTransport(t *testing.T) {
+       ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) {
+               w.Write([]byte("Hello"))
+       }))
+       defer ts.Close()
+
+       // TODO(bradfitz): add tests for skipping hostname checks too?
+       // would require a new cert for testing, and probably
+       // redundant with these tests.
+       for _, insecure := range []bool{true, false} {
+               tr := &Transport{
+                       TLSClientConfig: &tls.Config{
+                               InsecureSkipVerify: insecure,
+                       },
+               }
+               c := &Client{Transport: tr}
+               _, err := c.Get(ts.URL)
+               if (err == nil) != insecure {
+                       t.Errorf("insecure=%v: got unexpected err=%v", insecure, err)
+               }
+       }
+}
diff --git a/libgo/go/http/doc.go b/libgo/go/http/doc.go
new file mode 100644 (file)
index 0000000..9c47ac7
--- /dev/null
@@ -0,0 +1,79 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Package http provides HTTP client and server implementations.
+
+Get, Head, Post, and PostForm make HTTP requests:
+
+       resp, err := http.Get("http://example.com/")
+       ...
+       resp, err := http.Post("http://example.com/upload", "image/jpeg", &buf)
+       ...
+       resp, err := http.PostForm("http://example.com/form",
+               url.Values{"key": {"Value"}, "id": {"123"}})
+
+The client must close the response body when finished with it:
+
+       resp, err := http.Get("http://example.com/")
+       if err != nil {
+               // handle error
+       }
+       defer resp.Body.Close()
+       body, err := ioutil.ReadAll(resp.Body)
+       // ...
+
+For control over HTTP client headers, redirect policy, and other
+settings, create a Client:
+
+       client := &http.Client{
+               CheckRedirect: redirectPolicyFunc,
+       }
+
+       resp, err := client.Get("http://example.com")
+       // ...
+
+       req := http.NewRequest("GET", "http://example.com", nil)
+       req.Header.Add("If-None-Match", `W/"wyzzy"`)
+       resp, err := client.Do(req)
+       // ...
+
+For control over proxies, TLS configuration, keep-alives,
+compression, and other settings, create a Transport:
+
+       tr := &http.Transport{
+               TLSClientConfig:    &tls.Config{RootCAs: pool},
+               DisableCompression: true,
+       }
+       client := &http.Client{Transport: tr}
+       resp, err := client.Get("https://example.com")
+
+Clients and Transports are safe for concurrent use by multiple
+goroutines and for efficiency should only be created once and re-used.
+
+ListenAndServe starts an HTTP server with a given address and handler.
+The handler is usually nil, which means to use DefaultServeMux.
+Handle and HandleFunc add handlers to DefaultServeMux:
+
+       http.Handle("/foo", fooHandler)
+
+       http.HandleFunc("/bar", func(w http.ResponseWriter, r *http.Request) {
+               fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.RawPath))
+       })
+
+       log.Fatal(http.ListenAndServe(":8080", nil))
+
+More control over the server's behavior is available by creating a
+custom Server:
+
+       s := &http.Server{
+               Addr:           ":8080",
+               Handler:        myHandler,
+               ReadTimeout:    10e9,
+               WriteTimeout:   10e9,
+               MaxHeaderBytes: 1 << 20,
+       }
+       log.Fatal(s.ListenAndServe())
+*/
+package http
index 02317e0..78e07ec 100644 (file)
@@ -4,8 +4,6 @@
 
 // HTTP Request reading and parsing.
 
-// Package http implements parsing of HTTP requests, replies, and URLs and
-// provides an extensible HTTP server and a basic HTTP client.
 package http
 
 import (
index 0914af7..1d4433d 100644 (file)
@@ -2,6 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// HTTP client implementation. See RFC 2616.
+// 
+// This is the low-level Transport implementation of RoundTripper.
+// The high-level interface is in client.go.
+
 package http
 
 import (
@@ -357,8 +362,10 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, os.Error) {
                if err = conn.(*tls.Conn).Handshake(); err != nil {
                        return nil, err
                }
-               if err = conn.(*tls.Conn).VerifyHostname(cm.tlsHost()); err != nil {
-                       return nil, err
+               if t.TLSClientConfig == nil || !t.TLSClientConfig.InsecureSkipVerify {
+                       if err = conn.(*tls.Conn).VerifyHostname(cm.tlsHost()); err != nil {
+                               return nil, err
+                       }
                }
                pconn.conn = conn
        }
index c6dbd04..5169d1e 100644 (file)
@@ -11,9 +11,6 @@ import (
 )
 
 func setKernelSpecificSockopt(s syscall.Handle, f int) {
-       // Allow reuse of recently-used addresses and ports.
-       syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
-
        // Allow broadcast.
        syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)
 
index 88b5c0f..a3b8826 100644 (file)
@@ -9,227 +9,42 @@ const Version = "6.0.0"
 
 // Categories is the set of Unicode data tables.
 var Categories = map[string]*RangeTable{
-       "Lm": Lm,
-       "Ll": Ll,
        "C":  C,
-       "M":  M,
+       "Cc": Cc,
+       "Cf": Cf,
+       "Co": Co,
+       "Cs": Cs,
        "L":  L,
+       "Ll": Ll,
+       "Lm": Lm,
+       "Lo": Lo,
+       "Lt": Lt,
+       "Lu": Lu,
+       "M":  M,
+       "Mc": Mc,
+       "Me": Me,
+       "Mn": Mn,
        "N":  N,
+       "Nd": Nd,
+       "Nl": Nl,
+       "No": No,
        "P":  P,
+       "Pc": Pc,
+       "Pd": Pd,
+       "Pe": Pe,
+       "Pf": Pf,
+       "Pi": Pi,
+       "Po": Po,
+       "Ps": Ps,
        "S":  S,
+       "Sc": Sc,
+       "Sk": Sk,
+       "Sm": Sm,
+       "So": So,
        "Z":  Z,
-       "Me": Me,
-       "Mc": Mc,
-       "Mn": Mn,
        "Zl": Zl,
        "Zp": Zp,
        "Zs": Zs,
-       "Cs": Cs,
-       "Co": Co,
-       "Cf": Cf,
-       "Cc": Cc,
-       "Po": Po,
-       "Pi": Pi,
-       "Pf": Pf,
-       "Pe": Pe,
-       "Pd": Pd,
-       "Pc": Pc,
-       "Ps": Ps,
-       "Nd": Nd,
-       "Nl": Nl,
-       "No": No,
-       "So": So,
-       "Sm": Sm,
-       "Sk": Sk,
-       "Sc": Sc,
-       "Lu": Lu,
-       "Lt": Lt,
-       "Lo": Lo,
-}
-
-var _Lm = &RangeTable{
-       R16: []Range16{
-               {0x02b0, 0x02c1, 1},
-               {0x02c6, 0x02d1, 1},
-               {0x02e0, 0x02e4, 1},
-               {0x02ec, 0x02ee, 2},
-               {0x0374, 0x037a, 6},
-               {0x0559, 0x0640, 231},
-               {0x06e5, 0x06e6, 1},
-               {0x07f4, 0x07f5, 1},
-               {0x07fa, 0x081a, 32},
-               {0x0824, 0x0828, 4},
-               {0x0971, 0x0e46, 1237},
-               {0x0ec6, 0x10fc, 566},
-               {0x17d7, 0x1843, 108},
-               {0x1aa7, 0x1c78, 465},
-               {0x1c79, 0x1c7d, 1},
-               {0x1d2c, 0x1d61, 1},
-               {0x1d78, 0x1d9b, 35},
-               {0x1d9c, 0x1dbf, 1},
-               {0x2071, 0x207f, 14},
-               {0x2090, 0x209c, 1},
-               {0x2c7d, 0x2d6f, 242},
-               {0x2e2f, 0x3005, 470},
-               {0x3031, 0x3035, 1},
-               {0x303b, 0x309d, 98},
-               {0x309e, 0x30fc, 94},
-               {0x30fd, 0x30fe, 1},
-               {0xa015, 0xa4f8, 1251},
-               {0xa4f9, 0xa4fd, 1},
-               {0xa60c, 0xa67f, 115},
-               {0xa717, 0xa71f, 1},
-               {0xa770, 0xa788, 24},
-               {0xa9cf, 0xaa70, 161},
-               {0xaadd, 0xff70, 21651},
-               {0xff9e, 0xff9f, 1},
-       },
-}
-
-var _Ll = &RangeTable{
-       R16: []Range16{
-               {0x0061, 0x007a, 1},
-               {0x00aa, 0x00b5, 11},
-               {0x00ba, 0x00df, 37},
-               {0x00e0, 0x00f6, 1},
-               {0x00f8, 0x00ff, 1},
-               {0x0101, 0x0137, 2},
-               {0x0138, 0x0148, 2},
-               {0x0149, 0x0177, 2},
-               {0x017a, 0x017e, 2},
-               {0x017f, 0x0180, 1},
-               {0x0183, 0x0185, 2},
-               {0x0188, 0x018c, 4},
-               {0x018d, 0x0192, 5},
-               {0x0195, 0x0199, 4},
-               {0x019a, 0x019b, 1},
-               {0x019e, 0x01a1, 3},
-               {0x01a3, 0x01a5, 2},
-               {0x01a8, 0x01aa, 2},
-               {0x01ab, 0x01ad, 2},
-               {0x01b0, 0x01b4, 4},
-               {0x01b6, 0x01b9, 3},
-               {0x01ba, 0x01bd, 3},
-               {0x01be, 0x01bf, 1},
-               {0x01c6, 0x01cc, 3},
-               {0x01ce, 0x01dc, 2},
-               {0x01dd, 0x01ef, 2},
-               {0x01f0, 0x01f3, 3},
-               {0x01f5, 0x01f9, 4},
-               {0x01fb, 0x0233, 2},
-               {0x0234, 0x0239, 1},
-               {0x023c, 0x023f, 3},
-               {0x0240, 0x0242, 2},
-               {0x0247, 0x024f, 2},
-               {0x0250, 0x0293, 1},
-               {0x0295, 0x02af, 1},
-               {0x0371, 0x0373, 2},
-               {0x0377, 0x037b, 4},
-               {0x037c, 0x037d, 1},
-               {0x0390, 0x03ac, 28},
-               {0x03ad, 0x03ce, 1},
-               {0x03d0, 0x03d1, 1},
-               {0x03d5, 0x03d7, 1},
-               {0x03d9, 0x03ef, 2},
-               {0x03f0, 0x03f3, 1},
-               {0x03f5, 0x03fb, 3},
-               {0x03fc, 0x0430, 52},
-               {0x0431, 0x045f, 1},
-               {0x0461, 0x0481, 2},
-               {0x048b, 0x04bf, 2},
-               {0x04c2, 0x04ce, 2},
-               {0x04cf, 0x0527, 2},
-               {0x0561, 0x0587, 1},
-               {0x1d00, 0x1d2b, 1},
-               {0x1d62, 0x1d77, 1},
-               {0x1d79, 0x1d9a, 1},
-               {0x1e01, 0x1e95, 2},
-               {0x1e96, 0x1e9d, 1},
-               {0x1e9f, 0x1eff, 2},
-               {0x1f00, 0x1f07, 1},
-               {0x1f10, 0x1f15, 1},
-               {0x1f20, 0x1f27, 1},
-               {0x1f30, 0x1f37, 1},
-               {0x1f40, 0x1f45, 1},
-               {0x1f50, 0x1f57, 1},
-               {0x1f60, 0x1f67, 1},
-               {0x1f70, 0x1f7d, 1},
-               {0x1f80, 0x1f87, 1},
-               {0x1f90, 0x1f97, 1},
-               {0x1fa0, 0x1fa7, 1},
-               {0x1fb0, 0x1fb4, 1},
-               {0x1fb6, 0x1fb7, 1},
-               {0x1fbe, 0x1fc2, 4},
-               {0x1fc3, 0x1fc4, 1},
-               {0x1fc6, 0x1fc7, 1},
-               {0x1fd0, 0x1fd3, 1},
-               {0x1fd6, 0x1fd7, 1},
-               {0x1fe0, 0x1fe7, 1},
-               {0x1ff2, 0x1ff4, 1},
-               {0x1ff6, 0x1ff7, 1},
-               {0x210a, 0x210e, 4},
-               {0x210f, 0x2113, 4},
-               {0x212f, 0x2139, 5},
-               {0x213c, 0x213d, 1},
-               {0x2146, 0x2149, 1},
-               {0x214e, 0x2184, 54},
-               {0x2c30, 0x2c5e, 1},
-               {0x2c61, 0x2c65, 4},
-               {0x2c66, 0x2c6c, 2},
-               {0x2c71, 0x2c73, 2},
-               {0x2c74, 0x2c76, 2},
-               {0x2c77, 0x2c7c, 1},
-               {0x2c81, 0x2ce3, 2},
-               {0x2ce4, 0x2cec, 8},
-               {0x2cee, 0x2d00, 18},
-               {0x2d01, 0x2d25, 1},
-               {0xa641, 0xa66d, 2},
-               {0xa681, 0xa697, 2},
-               {0xa723, 0xa72f, 2},
-               {0xa730, 0xa731, 1},
-               {0xa733, 0xa771, 2},
-               {0xa772, 0xa778, 1},
-               {0xa77a, 0xa77c, 2},
-               {0xa77f, 0xa787, 2},
-               {0xa78c, 0xa78e, 2},
-               {0xa791, 0xa7a1, 16},
-               {0xa7a3, 0xa7a9, 2},
-               {0xa7fa, 0xfb00, 21254},
-               {0xfb01, 0xfb06, 1},
-               {0xfb13, 0xfb17, 1},
-               {0xff41, 0xff5a, 1},
-       },
-       R32: []Range32{
-               {0x10428, 0x1044f, 1},
-               {0x1d41a, 0x1d433, 1},
-               {0x1d44e, 0x1d454, 1},
-               {0x1d456, 0x1d467, 1},
-               {0x1d482, 0x1d49b, 1},
-               {0x1d4b6, 0x1d4b9, 1},
-               {0x1d4bb, 0x1d4bd, 2},
-               {0x1d4be, 0x1d4c3, 1},
-               {0x1d4c5, 0x1d4cf, 1},
-               {0x1d4ea, 0x1d503, 1},
-               {0x1d51e, 0x1d537, 1},
-               {0x1d552, 0x1d56b, 1},
-               {0x1d586, 0x1d59f, 1},
-               {0x1d5ba, 0x1d5d3, 1},
-               {0x1d5ee, 0x1d607, 1},
-               {0x1d622, 0x1d63b, 1},
-               {0x1d656, 0x1d66f, 1},
-               {0x1d68a, 0x1d6a5, 1},
-               {0x1d6c2, 0x1d6da, 1},
-               {0x1d6dc, 0x1d6e1, 1},
-               {0x1d6fc, 0x1d714, 1},
-               {0x1d716, 0x1d71b, 1},
-               {0x1d736, 0x1d74e, 1},
-               {0x1d750, 0x1d755, 1},
-               {0x1d770, 0x1d788, 1},
-               {0x1d78a, 0x1d78f, 1},
-               {0x1d7aa, 0x1d7c2, 1},
-               {0x1d7c4, 0x1d7c9, 1},
-               {0x1d7cb, 0x1d7cb, 1},
-       },
 }
 
 var _C = &RangeTable{
@@ -258,195 +73,47 @@ var _C = &RangeTable{
        },
 }
 
-var _M = &RangeTable{
+var _Cc = &RangeTable{
        R16: []Range16{
-               {0x0300, 0x036f, 1},
-               {0x0483, 0x0489, 1},
-               {0x0591, 0x05bd, 1},
-               {0x05bf, 0x05c1, 2},
-               {0x05c2, 0x05c4, 2},
-               {0x05c5, 0x05c7, 2},
-               {0x0610, 0x061a, 1},
-               {0x064b, 0x065f, 1},
-               {0x0670, 0x06d6, 102},
-               {0x06d7, 0x06dc, 1},
-               {0x06df, 0x06e4, 1},
-               {0x06e7, 0x06e8, 1},
-               {0x06ea, 0x06ed, 1},
-               {0x0711, 0x0730, 31},
-               {0x0731, 0x074a, 1},
-               {0x07a6, 0x07b0, 1},
-               {0x07eb, 0x07f3, 1},
-               {0x0816, 0x0819, 1},
-               {0x081b, 0x0823, 1},
-               {0x0825, 0x0827, 1},
-               {0x0829, 0x082d, 1},
-               {0x0859, 0x085b, 1},
-               {0x0900, 0x0903, 1},
-               {0x093a, 0x093c, 1},
-               {0x093e, 0x094f, 1},
-               {0x0951, 0x0957, 1},
-               {0x0962, 0x0963, 1},
-               {0x0981, 0x0983, 1},
-               {0x09bc, 0x09be, 2},
-               {0x09bf, 0x09c4, 1},
-               {0x09c7, 0x09c8, 1},
-               {0x09cb, 0x09cd, 1},
-               {0x09d7, 0x09e2, 11},
-               {0x09e3, 0x0a01, 30},
-               {0x0a02, 0x0a03, 1},
-               {0x0a3c, 0x0a3e, 2},
-               {0x0a3f, 0x0a42, 1},
-               {0x0a47, 0x0a48, 1},
-               {0x0a4b, 0x0a4d, 1},
-               {0x0a51, 0x0a70, 31},
-               {0x0a71, 0x0a75, 4},
-               {0x0a81, 0x0a83, 1},
-               {0x0abc, 0x0abe, 2},
-               {0x0abf, 0x0ac5, 1},
-               {0x0ac7, 0x0ac9, 1},
-               {0x0acb, 0x0acd, 1},
-               {0x0ae2, 0x0ae3, 1},
-               {0x0b01, 0x0b03, 1},
-               {0x0b3c, 0x0b3e, 2},
-               {0x0b3f, 0x0b44, 1},
-               {0x0b47, 0x0b48, 1},
-               {0x0b4b, 0x0b4d, 1},
-               {0x0b56, 0x0b57, 1},
-               {0x0b62, 0x0b63, 1},
-               {0x0b82, 0x0bbe, 60},
-               {0x0bbf, 0x0bc2, 1},
-               {0x0bc6, 0x0bc8, 1},
-               {0x0bca, 0x0bcd, 1},
-               {0x0bd7, 0x0c01, 42},
-               {0x0c02, 0x0c03, 1},
-               {0x0c3e, 0x0c44, 1},
-               {0x0c46, 0x0c48, 1},
-               {0x0c4a, 0x0c4d, 1},
-               {0x0c55, 0x0c56, 1},
-               {0x0c62, 0x0c63, 1},
-               {0x0c82, 0x0c83, 1},
-               {0x0cbc, 0x0cbe, 2},
-               {0x0cbf, 0x0cc4, 1},
-               {0x0cc6, 0x0cc8, 1},
-               {0x0cca, 0x0ccd, 1},
-               {0x0cd5, 0x0cd6, 1},
-               {0x0ce2, 0x0ce3, 1},
-               {0x0d02, 0x0d03, 1},
-               {0x0d3e, 0x0d44, 1},
-               {0x0d46, 0x0d48, 1},
-               {0x0d4a, 0x0d4d, 1},
-               {0x0d57, 0x0d62, 11},
-               {0x0d63, 0x0d82, 31},
-               {0x0d83, 0x0dca, 71},
-               {0x0dcf, 0x0dd4, 1},
-               {0x0dd6, 0x0dd8, 2},
-               {0x0dd9, 0x0ddf, 1},
-               {0x0df2, 0x0df3, 1},
-               {0x0e31, 0x0e34, 3},
-               {0x0e35, 0x0e3a, 1},
-               {0x0e47, 0x0e4e, 1},
-               {0x0eb1, 0x0eb4, 3},
-               {0x0eb5, 0x0eb9, 1},
-               {0x0ebb, 0x0ebc, 1},
-               {0x0ec8, 0x0ecd, 1},
-               {0x0f18, 0x0f19, 1},
-               {0x0f35, 0x0f39, 2},
-               {0x0f3e, 0x0f3f, 1},
-               {0x0f71, 0x0f84, 1},
-               {0x0f86, 0x0f87, 1},
-               {0x0f8d, 0x0f97, 1},
-               {0x0f99, 0x0fbc, 1},
-               {0x0fc6, 0x102b, 101},
-               {0x102c, 0x103e, 1},
-               {0x1056, 0x1059, 1},
-               {0x105e, 0x1060, 1},
-               {0x1062, 0x1064, 1},
-               {0x1067, 0x106d, 1},
-               {0x1071, 0x1074, 1},
-               {0x1082, 0x108d, 1},
-               {0x108f, 0x109a, 11},
-               {0x109b, 0x109d, 1},
-               {0x135d, 0x135f, 1},
-               {0x1712, 0x1714, 1},
-               {0x1732, 0x1734, 1},
-               {0x1752, 0x1753, 1},
-               {0x1772, 0x1773, 1},
-               {0x17b6, 0x17d3, 1},
-               {0x17dd, 0x180b, 46},
-               {0x180c, 0x180d, 1},
-               {0x18a9, 0x1920, 119},
-               {0x1921, 0x192b, 1},
-               {0x1930, 0x193b, 1},
-               {0x19b0, 0x19c0, 1},
-               {0x19c8, 0x19c9, 1},
-               {0x1a17, 0x1a1b, 1},
-               {0x1a55, 0x1a5e, 1},
-               {0x1a60, 0x1a7c, 1},
-               {0x1a7f, 0x1b00, 129},
-               {0x1b01, 0x1b04, 1},
-               {0x1b34, 0x1b44, 1},
-               {0x1b6b, 0x1b73, 1},
-               {0x1b80, 0x1b82, 1},
-               {0x1ba1, 0x1baa, 1},
-               {0x1be6, 0x1bf3, 1},
-               {0x1c24, 0x1c37, 1},
-               {0x1cd0, 0x1cd2, 1},
-               {0x1cd4, 0x1ce8, 1},
-               {0x1ced, 0x1cf2, 5},
-               {0x1dc0, 0x1de6, 1},
-               {0x1dfc, 0x1dff, 1},
-               {0x20d0, 0x20f0, 1},
-               {0x2cef, 0x2cf1, 1},
-               {0x2d7f, 0x2de0, 97},
-               {0x2de1, 0x2dff, 1},
-               {0x302a, 0x302f, 1},
-               {0x3099, 0x309a, 1},
-               {0xa66f, 0xa672, 1},
-               {0xa67c, 0xa67d, 1},
-               {0xa6f0, 0xa6f1, 1},
-               {0xa802, 0xa806, 4},
-               {0xa80b, 0xa823, 24},
-               {0xa824, 0xa827, 1},
-               {0xa880, 0xa881, 1},
-               {0xa8b4, 0xa8c4, 1},
-               {0xa8e0, 0xa8f1, 1},
-               {0xa926, 0xa92d, 1},
-               {0xa947, 0xa953, 1},
-               {0xa980, 0xa983, 1},
-               {0xa9b3, 0xa9c0, 1},
-               {0xaa29, 0xaa36, 1},
-               {0xaa43, 0xaa4c, 9},
-               {0xaa4d, 0xaa7b, 46},
-               {0xaab0, 0xaab2, 2},
-               {0xaab3, 0xaab4, 1},
-               {0xaab7, 0xaab8, 1},
-               {0xaabe, 0xaabf, 1},
-               {0xaac1, 0xabe3, 290},
-               {0xabe4, 0xabea, 1},
-               {0xabec, 0xabed, 1},
-               {0xfb1e, 0xfe00, 738},
-               {0xfe01, 0xfe0f, 1},
-               {0xfe20, 0xfe26, 1},
+               {0x0001, 0x001f, 1},
+               {0x007f, 0x009f, 1},
+       },
+}
+
+var _Cf = &RangeTable{
+       R16: []Range16{
+               {0x00ad, 0x0600, 1363},
+               {0x0601, 0x0603, 1},
+               {0x06dd, 0x070f, 50},
+               {0x17b4, 0x17b5, 1},
+               {0x200b, 0x200f, 1},
+               {0x202a, 0x202e, 1},
+               {0x2060, 0x2064, 1},
+               {0x206a, 0x206f, 1},
+               {0xfeff, 0xfff9, 250},
+               {0xfffa, 0xfffb, 1},
        },
        R32: []Range32{
-               {0x101fd, 0x10a01, 2052},
-               {0x10a02, 0x10a03, 1},
-               {0x10a05, 0x10a06, 1},
-               {0x10a0c, 0x10a0f, 1},
-               {0x10a38, 0x10a3a, 1},
-               {0x10a3f, 0x11000, 1473},
-               {0x11001, 0x11002, 1},
-               {0x11038, 0x11046, 1},
-               {0x11080, 0x11082, 1},
-               {0x110b0, 0x110ba, 1},
-               {0x1d165, 0x1d169, 1},
-               {0x1d16d, 0x1d172, 1},
-               {0x1d17b, 0x1d182, 1},
-               {0x1d185, 0x1d18b, 1},
-               {0x1d1aa, 0x1d1ad, 1},
-               {0x1d242, 0x1d244, 1},
-               {0xe0100, 0xe01ef, 1},
+               {0x110bd, 0x1d173, 49334},
+               {0x1d174, 0x1d17a, 1},
+               {0xe0001, 0xe0020, 31},
+               {0xe0021, 0xe007f, 1},
+       },
+}
+
+var _Co = &RangeTable{
+       R16: []Range16{
+               {0xe000, 0xf8ff, 1},
+       },
+       R32: []Range32{
+               {0xf0000, 0xffffd, 1},
+               {0x100000, 0x10fffd, 1},
+       },
+}
+
+var _Cs = &RangeTable{
+       R16: []Range16{
+               {0xd800, 0xdfff, 1},
        },
 }
 
@@ -871,1518 +538,192 @@ var _L = &RangeTable{
        },
 }
 
-var _N = &RangeTable{
+var _Ll = &RangeTable{
        R16: []Range16{
-               {0x0030, 0x0039, 1},
-               {0x00b2, 0x00b3, 1},
-               {0x00b9, 0x00bc, 3},
-               {0x00bd, 0x00be, 1},
-               {0x0660, 0x0669, 1},
-               {0x06f0, 0x06f9, 1},
-               {0x07c0, 0x07c9, 1},
-               {0x0966, 0x096f, 1},
-               {0x09e6, 0x09ef, 1},
-               {0x09f4, 0x09f9, 1},
-               {0x0a66, 0x0a6f, 1},
-               {0x0ae6, 0x0aef, 1},
-               {0x0b66, 0x0b6f, 1},
-               {0x0b72, 0x0b77, 1},
-               {0x0be6, 0x0bf2, 1},
-               {0x0c66, 0x0c6f, 1},
-               {0x0c78, 0x0c7e, 1},
-               {0x0ce6, 0x0cef, 1},
-               {0x0d66, 0x0d75, 1},
-               {0x0e50, 0x0e59, 1},
-               {0x0ed0, 0x0ed9, 1},
-               {0x0f20, 0x0f33, 1},
-               {0x1040, 0x1049, 1},
-               {0x1090, 0x1099, 1},
-               {0x1369, 0x137c, 1},
-               {0x16ee, 0x16f0, 1},
-               {0x17e0, 0x17e9, 1},
-               {0x17f0, 0x17f9, 1},
-               {0x1810, 0x1819, 1},
-               {0x1946, 0x194f, 1},
-               {0x19d0, 0x19da, 1},
-               {0x1a80, 0x1a89, 1},
-               {0x1a90, 0x1a99, 1},
-               {0x1b50, 0x1b59, 1},
-               {0x1bb0, 0x1bb9, 1},
-               {0x1c40, 0x1c49, 1},
-               {0x1c50, 0x1c59, 1},
-               {0x2070, 0x2074, 4},
-               {0x2075, 0x2079, 1},
-               {0x2080, 0x2089, 1},
-               {0x2150, 0x2182, 1},
-               {0x2185, 0x2189, 1},
-               {0x2460, 0x249b, 1},
-               {0x24ea, 0x24ff, 1},
-               {0x2776, 0x2793, 1},
-               {0x2cfd, 0x3007, 778},
-               {0x3021, 0x3029, 1},
-               {0x3038, 0x303a, 1},
-               {0x3192, 0x3195, 1},
-               {0x3220, 0x3229, 1},
-               {0x3251, 0x325f, 1},
-               {0x3280, 0x3289, 1},
-               {0x32b1, 0x32bf, 1},
-               {0xa620, 0xa629, 1},
-               {0xa6e6, 0xa6ef, 1},
-               {0xa830, 0xa835, 1},
-               {0xa8d0, 0xa8d9, 1},
-               {0xa900, 0xa909, 1},
-               {0xa9d0, 0xa9d9, 1},
-               {0xaa50, 0xaa59, 1},
-               {0xabf0, 0xabf9, 1},
-               {0xff10, 0xff19, 1},
+               {0x0061, 0x007a, 1},
+               {0x00aa, 0x00b5, 11},
+               {0x00ba, 0x00df, 37},
+               {0x00e0, 0x00f6, 1},
+               {0x00f8, 0x00ff, 1},
+               {0x0101, 0x0137, 2},
+               {0x0138, 0x0148, 2},
+               {0x0149, 0x0177, 2},
+               {0x017a, 0x017e, 2},
+               {0x017f, 0x0180, 1},
+               {0x0183, 0x0185, 2},
+               {0x0188, 0x018c, 4},
+               {0x018d, 0x0192, 5},
+               {0x0195, 0x0199, 4},
+               {0x019a, 0x019b, 1},
+               {0x019e, 0x01a1, 3},
+               {0x01a3, 0x01a5, 2},
+               {0x01a8, 0x01aa, 2},
+               {0x01ab, 0x01ad, 2},
+               {0x01b0, 0x01b4, 4},
+               {0x01b6, 0x01b9, 3},
+               {0x01ba, 0x01bd, 3},
+               {0x01be, 0x01bf, 1},
+               {0x01c6, 0x01cc, 3},
+               {0x01ce, 0x01dc, 2},
+               {0x01dd, 0x01ef, 2},
+               {0x01f0, 0x01f3, 3},
+               {0x01f5, 0x01f9, 4},
+               {0x01fb, 0x0233, 2},
+               {0x0234, 0x0239, 1},
+               {0x023c, 0x023f, 3},
+               {0x0240, 0x0242, 2},
+               {0x0247, 0x024f, 2},
+               {0x0250, 0x0293, 1},
+               {0x0295, 0x02af, 1},
+               {0x0371, 0x0373, 2},
+               {0x0377, 0x037b, 4},
+               {0x037c, 0x037d, 1},
+               {0x0390, 0x03ac, 28},
+               {0x03ad, 0x03ce, 1},
+               {0x03d0, 0x03d1, 1},
+               {0x03d5, 0x03d7, 1},
+               {0x03d9, 0x03ef, 2},
+               {0x03f0, 0x03f3, 1},
+               {0x03f5, 0x03fb, 3},
+               {0x03fc, 0x0430, 52},
+               {0x0431, 0x045f, 1},
+               {0x0461, 0x0481, 2},
+               {0x048b, 0x04bf, 2},
+               {0x04c2, 0x04ce, 2},
+               {0x04cf, 0x0527, 2},
+               {0x0561, 0x0587, 1},
+               {0x1d00, 0x1d2b, 1},
+               {0x1d62, 0x1d77, 1},
+               {0x1d79, 0x1d9a, 1},
+               {0x1e01, 0x1e95, 2},
+               {0x1e96, 0x1e9d, 1},
+               {0x1e9f, 0x1eff, 2},
+               {0x1f00, 0x1f07, 1},
+               {0x1f10, 0x1f15, 1},
+               {0x1f20, 0x1f27, 1},
+               {0x1f30, 0x1f37, 1},
+               {0x1f40, 0x1f45, 1},
+               {0x1f50, 0x1f57, 1},
+               {0x1f60, 0x1f67, 1},
+               {0x1f70, 0x1f7d, 1},
+               {0x1f80, 0x1f87, 1},
+               {0x1f90, 0x1f97, 1},
+               {0x1fa0, 0x1fa7, 1},
+               {0x1fb0, 0x1fb4, 1},
+               {0x1fb6, 0x1fb7, 1},
+               {0x1fbe, 0x1fc2, 4},
+               {0x1fc3, 0x1fc4, 1},
+               {0x1fc6, 0x1fc7, 1},
+               {0x1fd0, 0x1fd3, 1},
+               {0x1fd6, 0x1fd7, 1},
+               {0x1fe0, 0x1fe7, 1},
+               {0x1ff2, 0x1ff4, 1},
+               {0x1ff6, 0x1ff7, 1},
+               {0x210a, 0x210e, 4},
+               {0x210f, 0x2113, 4},
+               {0x212f, 0x2139, 5},
+               {0x213c, 0x213d, 1},
+               {0x2146, 0x2149, 1},
+               {0x214e, 0x2184, 54},
+               {0x2c30, 0x2c5e, 1},
+               {0x2c61, 0x2c65, 4},
+               {0x2c66, 0x2c6c, 2},
+               {0x2c71, 0x2c73, 2},
+               {0x2c74, 0x2c76, 2},
+               {0x2c77, 0x2c7c, 1},
+               {0x2c81, 0x2ce3, 2},
+               {0x2ce4, 0x2cec, 8},
+               {0x2cee, 0x2d00, 18},
+               {0x2d01, 0x2d25, 1},
+               {0xa641, 0xa66d, 2},
+               {0xa681, 0xa697, 2},
+               {0xa723, 0xa72f, 2},
+               {0xa730, 0xa731, 1},
+               {0xa733, 0xa771, 2},
+               {0xa772, 0xa778, 1},
+               {0xa77a, 0xa77c, 2},
+               {0xa77f, 0xa787, 2},
+               {0xa78c, 0xa78e, 2},
+               {0xa791, 0xa7a1, 16},
+               {0xa7a3, 0xa7a9, 2},
+               {0xa7fa, 0xfb00, 21254},
+               {0xfb01, 0xfb06, 1},
+               {0xfb13, 0xfb17, 1},
+               {0xff41, 0xff5a, 1},
        },
        R32: []Range32{
-               {0x10107, 0x10133, 1},
-               {0x10140, 0x10178, 1},
-               {0x1018a, 0x10320, 406},
-               {0x10321, 0x10323, 1},
-               {0x10341, 0x1034a, 9},
-               {0x103d1, 0x103d5, 1},
-               {0x104a0, 0x104a9, 1},
-               {0x10858, 0x1085f, 1},
-               {0x10916, 0x1091b, 1},
-               {0x10a40, 0x10a47, 1},
-               {0x10a7d, 0x10a7e, 1},
-               {0x10b58, 0x10b5f, 1},
-               {0x10b78, 0x10b7f, 1},
-               {0x10e60, 0x10e7e, 1},
-               {0x11052, 0x1106f, 1},
-               {0x12400, 0x12462, 1},
-               {0x1d360, 0x1d371, 1},
-               {0x1d7ce, 0x1d7ff, 1},
-               {0x1f100, 0x1f10a, 1},
+               {0x10428, 0x1044f, 1},
+               {0x1d41a, 0x1d433, 1},
+               {0x1d44e, 0x1d454, 1},
+               {0x1d456, 0x1d467, 1},
+               {0x1d482, 0x1d49b, 1},
+               {0x1d4b6, 0x1d4b9, 1},
+               {0x1d4bb, 0x1d4bd, 2},
+               {0x1d4be, 0x1d4c3, 1},
+               {0x1d4c5, 0x1d4cf, 1},
+               {0x1d4ea, 0x1d503, 1},
+               {0x1d51e, 0x1d537, 1},
+               {0x1d552, 0x1d56b, 1},
+               {0x1d586, 0x1d59f, 1},
+               {0x1d5ba, 0x1d5d3, 1},
+               {0x1d5ee, 0x1d607, 1},
+               {0x1d622, 0x1d63b, 1},
+               {0x1d656, 0x1d66f, 1},
+               {0x1d68a, 0x1d6a5, 1},
+               {0x1d6c2, 0x1d6da, 1},
+               {0x1d6dc, 0x1d6e1, 1},
+               {0x1d6fc, 0x1d714, 1},
+               {0x1d716, 0x1d71b, 1},
+               {0x1d736, 0x1d74e, 1},
+               {0x1d750, 0x1d755, 1},
+               {0x1d770, 0x1d788, 1},
+               {0x1d78a, 0x1d78f, 1},
+               {0x1d7aa, 0x1d7c2, 1},
+               {0x1d7c4, 0x1d7c9, 1},
+               {0x1d7cb, 0x1d7cb, 1},
        },
 }
 
-var _P = &RangeTable{
+var _Lm = &RangeTable{
        R16: []Range16{
-               {0x0021, 0x0023, 1},
-               {0x0025, 0x002a, 1},
-               {0x002c, 0x002f, 1},
-               {0x003a, 0x003b, 1},
-               {0x003f, 0x0040, 1},
-               {0x005b, 0x005d, 1},
-               {0x005f, 0x007b, 28},
-               {0x007d, 0x00a1, 36},
-               {0x00ab, 0x00b7, 12},
-               {0x00bb, 0x00bf, 4},
-               {0x037e, 0x0387, 9},
-               {0x055a, 0x055f, 1},
-               {0x0589, 0x058a, 1},
-               {0x05be, 0x05c0, 2},
-               {0x05c3, 0x05c6, 3},
-               {0x05f3, 0x05f4, 1},
-               {0x0609, 0x060a, 1},
-               {0x060c, 0x060d, 1},
-               {0x061b, 0x061e, 3},
-               {0x061f, 0x066a, 75},
-               {0x066b, 0x066d, 1},
-               {0x06d4, 0x0700, 44},
-               {0x0701, 0x070d, 1},
-               {0x07f7, 0x07f9, 1},
-               {0x0830, 0x083e, 1},
-               {0x085e, 0x0964, 262},
-               {0x0965, 0x0970, 11},
-               {0x0df4, 0x0e4f, 91},
-               {0x0e5a, 0x0e5b, 1},
-               {0x0f04, 0x0f12, 1},
-               {0x0f3a, 0x0f3d, 1},
-               {0x0f85, 0x0fd0, 75},
-               {0x0fd1, 0x0fd4, 1},
-               {0x0fd9, 0x0fda, 1},
-               {0x104a, 0x104f, 1},
-               {0x10fb, 0x1361, 614},
-               {0x1362, 0x1368, 1},
-               {0x1400, 0x166d, 621},
-               {0x166e, 0x169b, 45},
-               {0x169c, 0x16eb, 79},
-               {0x16ec, 0x16ed, 1},
-               {0x1735, 0x1736, 1},
-               {0x17d4, 0x17d6, 1},
-               {0x17d8, 0x17da, 1},
-               {0x1800, 0x180a, 1},
-               {0x1944, 0x1945, 1},
-               {0x1a1e, 0x1a1f, 1},
-               {0x1aa0, 0x1aa6, 1},
-               {0x1aa8, 0x1aad, 1},
-               {0x1b5a, 0x1b60, 1},
-               {0x1bfc, 0x1bff, 1},
-               {0x1c3b, 0x1c3f, 1},
-               {0x1c7e, 0x1c7f, 1},
-               {0x1cd3, 0x2010, 829},
-               {0x2011, 0x2027, 1},
-               {0x2030, 0x2043, 1},
-               {0x2045, 0x2051, 1},
-               {0x2053, 0x205e, 1},
-               {0x207d, 0x207e, 1},
-               {0x208d, 0x208e, 1},
-               {0x2329, 0x232a, 1},
-               {0x2768, 0x2775, 1},
-               {0x27c5, 0x27c6, 1},
-               {0x27e6, 0x27ef, 1},
-               {0x2983, 0x2998, 1},
-               {0x29d8, 0x29db, 1},
-               {0x29fc, 0x29fd, 1},
-               {0x2cf9, 0x2cfc, 1},
-               {0x2cfe, 0x2cff, 1},
-               {0x2d70, 0x2e00, 144},
-               {0x2e01, 0x2e2e, 1},
-               {0x2e30, 0x2e31, 1},
-               {0x3001, 0x3003, 1},
-               {0x3008, 0x3011, 1},
-               {0x3014, 0x301f, 1},
-               {0x3030, 0x303d, 13},
-               {0x30a0, 0x30fb, 91},
-               {0xa4fe, 0xa4ff, 1},
-               {0xa60d, 0xa60f, 1},
-               {0xa673, 0xa67e, 11},
-               {0xa6f2, 0xa6f7, 1},
-               {0xa874, 0xa877, 1},
-               {0xa8ce, 0xa8cf, 1},
-               {0xa8f8, 0xa8fa, 1},
-               {0xa92e, 0xa92f, 1},
-               {0xa95f, 0xa9c1, 98},
-               {0xa9c2, 0xa9cd, 1},
-               {0xa9de, 0xa9df, 1},
-               {0xaa5c, 0xaa5f, 1},
-               {0xaade, 0xaadf, 1},
-               {0xabeb, 0xfd3e, 20819},
-               {0xfd3f, 0xfe10, 209},
-               {0xfe11, 0xfe19, 1},
-               {0xfe30, 0xfe52, 1},
-               {0xfe54, 0xfe61, 1},
-               {0xfe63, 0xfe68, 5},
-               {0xfe6a, 0xfe6b, 1},
-               {0xff01, 0xff03, 1},
-               {0xff05, 0xff0a, 1},
-               {0xff0c, 0xff0f, 1},
-               {0xff1a, 0xff1b, 1},
-               {0xff1f, 0xff20, 1},
-               {0xff3b, 0xff3d, 1},
-               {0xff3f, 0xff5b, 28},
-               {0xff5d, 0xff5f, 2},
-               {0xff60, 0xff65, 1},
-       },
-       R32: []Range32{
-               {0x10100, 0x10101, 1},
-               {0x1039f, 0x103d0, 49},
-               {0x10857, 0x1091f, 200},
-               {0x1093f, 0x10a50, 273},
-               {0x10a51, 0x10a58, 1},
-               {0x10a7f, 0x10b39, 186},
-               {0x10b3a, 0x10b3f, 1},
-               {0x11047, 0x1104d, 1},
-               {0x110bb, 0x110bc, 1},
-               {0x110be, 0x110c1, 1},
-               {0x12470, 0x12473, 1},
+               {0x02b0, 0x02c1, 1},
+               {0x02c6, 0x02d1, 1},
+               {0x02e0, 0x02e4, 1},
+               {0x02ec, 0x02ee, 2},
+               {0x0374, 0x037a, 6},
+               {0x0559, 0x0640, 231},
+               {0x06e5, 0x06e6, 1},
+               {0x07f4, 0x07f5, 1},
+               {0x07fa, 0x081a, 32},
+               {0x0824, 0x0828, 4},
+               {0x0971, 0x0e46, 1237},
+               {0x0ec6, 0x10fc, 566},
+               {0x17d7, 0x1843, 108},
+               {0x1aa7, 0x1c78, 465},
+               {0x1c79, 0x1c7d, 1},
+               {0x1d2c, 0x1d61, 1},
+               {0x1d78, 0x1d9b, 35},
+               {0x1d9c, 0x1dbf, 1},
+               {0x2071, 0x207f, 14},
+               {0x2090, 0x209c, 1},
+               {0x2c7d, 0x2d6f, 242},
+               {0x2e2f, 0x3005, 470},
+               {0x3031, 0x3035, 1},
+               {0x303b, 0x309d, 98},
+               {0x309e, 0x30fc, 94},
+               {0x30fd, 0x30fe, 1},
+               {0xa015, 0xa4f8, 1251},
+               {0xa4f9, 0xa4fd, 1},
+               {0xa60c, 0xa67f, 115},
+               {0xa717, 0xa71f, 1},
+               {0xa770, 0xa788, 24},
+               {0xa9cf, 0xaa70, 161},
+               {0xaadd, 0xff70, 21651},
+               {0xff9e, 0xff9f, 1},
        },
 }
 
-var _S = &RangeTable{
-       R16: []Range16{
-               {0x0024, 0x002b, 7},
-               {0x003c, 0x003e, 1},
-               {0x005e, 0x0060, 2},
-               {0x007c, 0x007e, 2},
-               {0x00a2, 0x00a9, 1},
-               {0x00ac, 0x00ae, 2},
-               {0x00af, 0x00b1, 1},
-               {0x00b4, 0x00b8, 2},
-               {0x00d7, 0x00f7, 32},
-               {0x02c2, 0x02c5, 1},
-               {0x02d2, 0x02df, 1},
-               {0x02e5, 0x02eb, 1},
-               {0x02ed, 0x02ef, 2},
-               {0x02f0, 0x02ff, 1},
-               {0x0375, 0x0384, 15},
-               {0x0385, 0x03f6, 113},
-               {0x0482, 0x0606, 388},
-               {0x0607, 0x0608, 1},
-               {0x060b, 0x060e, 3},
-               {0x060f, 0x06de, 207},
-               {0x06e9, 0x06fd, 20},
-               {0x06fe, 0x07f6, 248},
-               {0x09f2, 0x09f3, 1},
-               {0x09fa, 0x09fb, 1},
-               {0x0af1, 0x0b70, 127},
-               {0x0bf3, 0x0bfa, 1},
-               {0x0c7f, 0x0d79, 250},
-               {0x0e3f, 0x0f01, 194},
-               {0x0f02, 0x0f03, 1},
-               {0x0f13, 0x0f17, 1},
-               {0x0f1a, 0x0f1f, 1},
-               {0x0f34, 0x0f38, 2},
-               {0x0fbe, 0x0fc5, 1},
-               {0x0fc7, 0x0fcc, 1},
-               {0x0fce, 0x0fcf, 1},
-               {0x0fd5, 0x0fd8, 1},
-               {0x109e, 0x109f, 1},
-               {0x1360, 0x1390, 48},
-               {0x1391, 0x1399, 1},
-               {0x17db, 0x1940, 357},
-               {0x19de, 0x19ff, 1},
-               {0x1b61, 0x1b6a, 1},
-               {0x1b74, 0x1b7c, 1},
-               {0x1fbd, 0x1fbf, 2},
-               {0x1fc0, 0x1fc1, 1},
-               {0x1fcd, 0x1fcf, 1},
-               {0x1fdd, 0x1fdf, 1},
-               {0x1fed, 0x1fef, 1},
-               {0x1ffd, 0x1ffe, 1},
-               {0x2044, 0x2052, 14},
-               {0x207a, 0x207c, 1},
-               {0x208a, 0x208c, 1},
-               {0x20a0, 0x20b9, 1},
-               {0x2100, 0x2101, 1},
-               {0x2103, 0x2106, 1},
-               {0x2108, 0x2109, 1},
-               {0x2114, 0x2116, 2},
-               {0x2117, 0x2118, 1},
-               {0x211e, 0x2123, 1},
-               {0x2125, 0x2129, 2},
-               {0x212e, 0x213a, 12},
-               {0x213b, 0x2140, 5},
-               {0x2141, 0x2144, 1},
-               {0x214a, 0x214d, 1},
-               {0x214f, 0x2190, 65},
-               {0x2191, 0x2328, 1},
-               {0x232b, 0x23f3, 1},
-               {0x2400, 0x2426, 1},
-               {0x2440, 0x244a, 1},
-               {0x249c, 0x24e9, 1},
-               {0x2500, 0x26ff, 1},
-               {0x2701, 0x2767, 1},
-               {0x2794, 0x27c4, 1},
-               {0x27c7, 0x27ca, 1},
-               {0x27cc, 0x27ce, 2},
-               {0x27cf, 0x27e5, 1},
-               {0x27f0, 0x2982, 1},
-               {0x2999, 0x29d7, 1},
-               {0x29dc, 0x29fb, 1},
-               {0x29fe, 0x2b4c, 1},
-               {0x2b50, 0x2b59, 1},
-               {0x2ce5, 0x2cea, 1},
-               {0x2e80, 0x2e99, 1},
-               {0x2e9b, 0x2ef3, 1},
-               {0x2f00, 0x2fd5, 1},
-               {0x2ff0, 0x2ffb, 1},
-               {0x3004, 0x3012, 14},
-               {0x3013, 0x3020, 13},
-               {0x3036, 0x3037, 1},
-               {0x303e, 0x303f, 1},
-               {0x309b, 0x309c, 1},
-               {0x3190, 0x3191, 1},
-               {0x3196, 0x319f, 1},
-               {0x31c0, 0x31e3, 1},
-               {0x3200, 0x321e, 1},
-               {0x322a, 0x3250, 1},
-               {0x3260, 0x327f, 1},
-               {0x328a, 0x32b0, 1},
-               {0x32c0, 0x32fe, 1},
-               {0x3300, 0x33ff, 1},
-               {0x4dc0, 0x4dff, 1},
-               {0xa490, 0xa4c6, 1},
-               {0xa700, 0xa716, 1},
-               {0xa720, 0xa721, 1},
-               {0xa789, 0xa78a, 1},
-               {0xa828, 0xa82b, 1},
-               {0xa836, 0xa839, 1},
-               {0xaa77, 0xaa79, 1},
-               {0xfb29, 0xfbb2, 137},
-               {0xfbb3, 0xfbc1, 1},
-               {0xfdfc, 0xfdfd, 1},
-               {0xfe62, 0xfe64, 2},
-               {0xfe65, 0xfe66, 1},
-               {0xfe69, 0xff04, 155},
-               {0xff0b, 0xff1c, 17},
-               {0xff1d, 0xff1e, 1},
-               {0xff3e, 0xff40, 2},
-               {0xff5c, 0xff5e, 2},
-               {0xffe0, 0xffe6, 1},
-               {0xffe8, 0xffee, 1},
-               {0xfffc, 0xfffd, 1},
-       },
-       R32: []Range32{
-               {0x10102, 0x10137, 53},
-               {0x10138, 0x1013f, 1},
-               {0x10179, 0x10189, 1},
-               {0x10190, 0x1019b, 1},
-               {0x101d0, 0x101fc, 1},
-               {0x1d000, 0x1d0f5, 1},
-               {0x1d100, 0x1d126, 1},
-               {0x1d129, 0x1d164, 1},
-               {0x1d16a, 0x1d16c, 1},
-               {0x1d183, 0x1d184, 1},
-               {0x1d18c, 0x1d1a9, 1},
-               {0x1d1ae, 0x1d1dd, 1},
-               {0x1d200, 0x1d241, 1},
-               {0x1d245, 0x1d300, 187},
-               {0x1d301, 0x1d356, 1},
-               {0x1d6c1, 0x1d6db, 26},
-               {0x1d6fb, 0x1d715, 26},
-               {0x1d735, 0x1d74f, 26},
-               {0x1d76f, 0x1d789, 26},
-               {0x1d7a9, 0x1d7c3, 26},
-               {0x1f000, 0x1f02b, 1},
-               {0x1f030, 0x1f093, 1},
-               {0x1f0a0, 0x1f0ae, 1},
-               {0x1f0b1, 0x1f0be, 1},
-               {0x1f0c1, 0x1f0cf, 1},
-               {0x1f0d1, 0x1f0df, 1},
-               {0x1f110, 0x1f12e, 1},
-               {0x1f130, 0x1f169, 1},
-               {0x1f170, 0x1f19a, 1},
-               {0x1f1e6, 0x1f202, 1},
-               {0x1f210, 0x1f23a, 1},
-               {0x1f240, 0x1f248, 1},
-               {0x1f250, 0x1f251, 1},
-               {0x1f300, 0x1f320, 1},
-               {0x1f330, 0x1f335, 1},
-               {0x1f337, 0x1f37c, 1},
-               {0x1f380, 0x1f393, 1},
-               {0x1f3a0, 0x1f3c4, 1},
-               {0x1f3c6, 0x1f3ca, 1},
-               {0x1f3e0, 0x1f3f0, 1},
-               {0x1f400, 0x1f43e, 1},
-               {0x1f440, 0x1f442, 2},
-               {0x1f443, 0x1f4f7, 1},
-               {0x1f4f9, 0x1f4fc, 1},
-               {0x1f500, 0x1f53d, 1},
-               {0x1f550, 0x1f567, 1},
-               {0x1f5fb, 0x1f5ff, 1},
-               {0x1f601, 0x1f610, 1},
-               {0x1f612, 0x1f614, 1},
-               {0x1f616, 0x1f61c, 2},
-               {0x1f61d, 0x1f61e, 1},
-               {0x1f620, 0x1f625, 1},
-               {0x1f628, 0x1f62b, 1},
-               {0x1f62d, 0x1f630, 3},
-               {0x1f631, 0x1f633, 1},
-               {0x1f635, 0x1f640, 1},
-               {0x1f645, 0x1f64f, 1},
-               {0x1f680, 0x1f6c5, 1},
-               {0x1f700, 0x1f773, 1},
-       },
-}
-
-var _Z = &RangeTable{
-       R16: []Range16{
-               {0x0020, 0x00a0, 128},
-               {0x1680, 0x180e, 398},
-               {0x2000, 0x200a, 1},
-               {0x2028, 0x2029, 1},
-               {0x202f, 0x205f, 48},
-               {0x3000, 0x3000, 1},
-       },
-}
-
-var _Me = &RangeTable{
-       R16: []Range16{
-               {0x0488, 0x0489, 1},
-               {0x20dd, 0x20e0, 1},
-               {0x20e2, 0x20e4, 1},
-               {0xa670, 0xa672, 1},
-       },
-}
-
-var _Mc = &RangeTable{
-       R16: []Range16{
-               {0x0903, 0x093b, 56},
-               {0x093e, 0x0940, 1},
-               {0x0949, 0x094c, 1},
-               {0x094e, 0x094f, 1},
-               {0x0982, 0x0983, 1},
-               {0x09be, 0x09c0, 1},
-               {0x09c7, 0x09c8, 1},
-               {0x09cb, 0x09cc, 1},
-               {0x09d7, 0x0a03, 44},
-               {0x0a3e, 0x0a40, 1},
-               {0x0a83, 0x0abe, 59},
-               {0x0abf, 0x0ac0, 1},
-               {0x0ac9, 0x0acb, 2},
-               {0x0acc, 0x0b02, 54},
-               {0x0b03, 0x0b3e, 59},
-               {0x0b40, 0x0b47, 7},
-               {0x0b48, 0x0b4b, 3},
-               {0x0b4c, 0x0b57, 11},
-               {0x0bbe, 0x0bbf, 1},
-               {0x0bc1, 0x0bc2, 1},
-               {0x0bc6, 0x0bc8, 1},
-               {0x0bca, 0x0bcc, 1},
-               {0x0bd7, 0x0c01, 42},
-               {0x0c02, 0x0c03, 1},
-               {0x0c41, 0x0c44, 1},
-               {0x0c82, 0x0c83, 1},
-               {0x0cbe, 0x0cc0, 2},
-               {0x0cc1, 0x0cc4, 1},
-               {0x0cc7, 0x0cc8, 1},
-               {0x0cca, 0x0ccb, 1},
-               {0x0cd5, 0x0cd6, 1},
-               {0x0d02, 0x0d03, 1},
-               {0x0d3e, 0x0d40, 1},
-               {0x0d46, 0x0d48, 1},
-               {0x0d4a, 0x0d4c, 1},
-               {0x0d57, 0x0d82, 43},
-               {0x0d83, 0x0dcf, 76},
-               {0x0dd0, 0x0dd1, 1},
-               {0x0dd8, 0x0ddf, 1},
-               {0x0df2, 0x0df3, 1},
-               {0x0f3e, 0x0f3f, 1},
-               {0x0f7f, 0x102b, 172},
-               {0x102c, 0x1031, 5},
-               {0x1038, 0x103b, 3},
-               {0x103c, 0x1056, 26},
-               {0x1057, 0x1062, 11},
-               {0x1063, 0x1064, 1},
-               {0x1067, 0x106d, 1},
-               {0x1083, 0x1084, 1},
-               {0x1087, 0x108c, 1},
-               {0x108f, 0x109a, 11},
-               {0x109b, 0x109c, 1},
-               {0x17b6, 0x17be, 8},
-               {0x17bf, 0x17c5, 1},
-               {0x17c7, 0x17c8, 1},
-               {0x1923, 0x1926, 1},
-               {0x1929, 0x192b, 1},
-               {0x1930, 0x1931, 1},
-               {0x1933, 0x1938, 1},
-               {0x19b0, 0x19c0, 1},
-               {0x19c8, 0x19c9, 1},
-               {0x1a19, 0x1a1b, 1},
-               {0x1a55, 0x1a57, 2},
-               {0x1a61, 0x1a63, 2},
-               {0x1a64, 0x1a6d, 9},
-               {0x1a6e, 0x1a72, 1},
-               {0x1b04, 0x1b35, 49},
-               {0x1b3b, 0x1b3d, 2},
-               {0x1b3e, 0x1b41, 1},
-               {0x1b43, 0x1b44, 1},
-               {0x1b82, 0x1ba1, 31},
-               {0x1ba6, 0x1ba7, 1},
-               {0x1baa, 0x1be7, 61},
-               {0x1bea, 0x1bec, 1},
-               {0x1bee, 0x1bf2, 4},
-               {0x1bf3, 0x1c24, 49},
-               {0x1c25, 0x1c2b, 1},
-               {0x1c34, 0x1c35, 1},
-               {0x1ce1, 0x1cf2, 17},
-               {0xa823, 0xa824, 1},
-               {0xa827, 0xa880, 89},
-               {0xa881, 0xa8b4, 51},
-               {0xa8b5, 0xa8c3, 1},
-               {0xa952, 0xa953, 1},
-               {0xa983, 0xa9b4, 49},
-               {0xa9b5, 0xa9ba, 5},
-               {0xa9bb, 0xa9bd, 2},
-               {0xa9be, 0xa9c0, 1},
-               {0xaa2f, 0xaa30, 1},
-               {0xaa33, 0xaa34, 1},
-               {0xaa4d, 0xaa7b, 46},
-               {0xabe3, 0xabe4, 1},
-               {0xabe6, 0xabe7, 1},
-               {0xabe9, 0xabea, 1},
-               {0xabec, 0xabec, 1},
-       },
-       R32: []Range32{
-               {0x11000, 0x11000, 1},
-               {0x11002, 0x11082, 128},
-               {0x110b0, 0x110b2, 1},
-               {0x110b7, 0x110b8, 1},
-               {0x1d165, 0x1d166, 1},
-               {0x1d16d, 0x1d172, 1},
-       },
-}
-
-var _Mn = &RangeTable{
-       R16: []Range16{
-               {0x0300, 0x036f, 1},
-               {0x0483, 0x0487, 1},
-               {0x0591, 0x05bd, 1},
-               {0x05bf, 0x05c1, 2},
-               {0x05c2, 0x05c4, 2},
-               {0x05c5, 0x05c7, 2},
-               {0x0610, 0x061a, 1},
-               {0x064b, 0x065f, 1},
-               {0x0670, 0x06d6, 102},
-               {0x06d7, 0x06dc, 1},
-               {0x06df, 0x06e4, 1},
-               {0x06e7, 0x06e8, 1},
-               {0x06ea, 0x06ed, 1},
-               {0x0711, 0x0730, 31},
-               {0x0731, 0x074a, 1},
-               {0x07a6, 0x07b0, 1},
-               {0x07eb, 0x07f3, 1},
-               {0x0816, 0x0819, 1},
-               {0x081b, 0x0823, 1},
-               {0x0825, 0x0827, 1},
-               {0x0829, 0x082d, 1},
-               {0x0859, 0x085b, 1},
-               {0x0900, 0x0902, 1},
-               {0x093a, 0x093c, 2},
-               {0x0941, 0x0948, 1},
-               {0x094d, 0x0951, 4},
-               {0x0952, 0x0957, 1},
-               {0x0962, 0x0963, 1},
-               {0x0981, 0x09bc, 59},
-               {0x09c1, 0x09c4, 1},
-               {0x09cd, 0x09e2, 21},
-               {0x09e3, 0x0a01, 30},
-               {0x0a02, 0x0a3c, 58},
-               {0x0a41, 0x0a42, 1},
-               {0x0a47, 0x0a48, 1},
-               {0x0a4b, 0x0a4d, 1},
-               {0x0a51, 0x0a70, 31},
-               {0x0a71, 0x0a75, 4},
-               {0x0a81, 0x0a82, 1},
-               {0x0abc, 0x0ac1, 5},
-               {0x0ac2, 0x0ac5, 1},
-               {0x0ac7, 0x0ac8, 1},
-               {0x0acd, 0x0ae2, 21},
-               {0x0ae3, 0x0b01, 30},
-               {0x0b3c, 0x0b3f, 3},
-               {0x0b41, 0x0b44, 1},
-               {0x0b4d, 0x0b56, 9},
-               {0x0b62, 0x0b63, 1},
-               {0x0b82, 0x0bc0, 62},
-               {0x0bcd, 0x0c3e, 113},
-               {0x0c3f, 0x0c40, 1},
-               {0x0c46, 0x0c48, 1},
-               {0x0c4a, 0x0c4d, 1},
-               {0x0c55, 0x0c56, 1},
-               {0x0c62, 0x0c63, 1},
-               {0x0cbc, 0x0cbf, 3},
-               {0x0cc6, 0x0ccc, 6},
-               {0x0ccd, 0x0ce2, 21},
-               {0x0ce3, 0x0d41, 94},
-               {0x0d42, 0x0d44, 1},
-               {0x0d4d, 0x0d62, 21},
-               {0x0d63, 0x0dca, 103},
-               {0x0dd2, 0x0dd4, 1},
-               {0x0dd6, 0x0e31, 91},
-               {0x0e34, 0x0e3a, 1},
-               {0x0e47, 0x0e4e, 1},
-               {0x0eb1, 0x0eb4, 3},
-               {0x0eb5, 0x0eb9, 1},
-               {0x0ebb, 0x0ebc, 1},
-               {0x0ec8, 0x0ecd, 1},
-               {0x0f18, 0x0f19, 1},
-               {0x0f35, 0x0f39, 2},
-               {0x0f71, 0x0f7e, 1},
-               {0x0f80, 0x0f84, 1},
-               {0x0f86, 0x0f87, 1},
-               {0x0f8d, 0x0f97, 1},
-               {0x0f99, 0x0fbc, 1},
-               {0x0fc6, 0x102d, 103},
-               {0x102e, 0x1030, 1},
-               {0x1032, 0x1037, 1},
-               {0x1039, 0x103a, 1},
-               {0x103d, 0x103e, 1},
-               {0x1058, 0x1059, 1},
-               {0x105e, 0x1060, 1},
-               {0x1071, 0x1074, 1},
-               {0x1082, 0x1085, 3},
-               {0x1086, 0x108d, 7},
-               {0x109d, 0x135d, 704},
-               {0x135e, 0x135f, 1},
-               {0x1712, 0x1714, 1},
-               {0x1732, 0x1734, 1},
-               {0x1752, 0x1753, 1},
-               {0x1772, 0x1773, 1},
-               {0x17b7, 0x17bd, 1},
-               {0x17c6, 0x17c9, 3},
-               {0x17ca, 0x17d3, 1},
-               {0x17dd, 0x180b, 46},
-               {0x180c, 0x180d, 1},
-               {0x18a9, 0x1920, 119},
-               {0x1921, 0x1922, 1},
-               {0x1927, 0x1928, 1},
-               {0x1932, 0x1939, 7},
-               {0x193a, 0x193b, 1},
-               {0x1a17, 0x1a18, 1},
-               {0x1a56, 0x1a58, 2},
-               {0x1a59, 0x1a5e, 1},
-               {0x1a60, 0x1a62, 2},
-               {0x1a65, 0x1a6c, 1},
-               {0x1a73, 0x1a7c, 1},
-               {0x1a7f, 0x1b00, 129},
-               {0x1b01, 0x1b03, 1},
-               {0x1b34, 0x1b36, 2},
-               {0x1b37, 0x1b3a, 1},
-               {0x1b3c, 0x1b42, 6},
-               {0x1b6b, 0x1b73, 1},
-               {0x1b80, 0x1b81, 1},
-               {0x1ba2, 0x1ba5, 1},
-               {0x1ba8, 0x1ba9, 1},
-               {0x1be6, 0x1be8, 2},
-               {0x1be9, 0x1bed, 4},
-               {0x1bef, 0x1bf1, 1},
-               {0x1c2c, 0x1c33, 1},
-               {0x1c36, 0x1c37, 1},
-               {0x1cd0, 0x1cd2, 1},
-               {0x1cd4, 0x1ce0, 1},
-               {0x1ce2, 0x1ce8, 1},
-               {0x1ced, 0x1dc0, 211},
-               {0x1dc1, 0x1de6, 1},
-               {0x1dfc, 0x1dff, 1},
-               {0x20d0, 0x20dc, 1},
-               {0x20e1, 0x20e5, 4},
-               {0x20e6, 0x20f0, 1},
-               {0x2cef, 0x2cf1, 1},
-               {0x2d7f, 0x2de0, 97},
-               {0x2de1, 0x2dff, 1},
-               {0x302a, 0x302f, 1},
-               {0x3099, 0x309a, 1},
-               {0xa66f, 0xa67c, 13},
-               {0xa67d, 0xa6f0, 115},
-               {0xa6f1, 0xa802, 273},
-               {0xa806, 0xa80b, 5},
-               {0xa825, 0xa826, 1},
-               {0xa8c4, 0xa8e0, 28},
-               {0xa8e1, 0xa8f1, 1},
-               {0xa926, 0xa92d, 1},
-               {0xa947, 0xa951, 1},
-               {0xa980, 0xa982, 1},
-               {0xa9b3, 0xa9b6, 3},
-               {0xa9b7, 0xa9b9, 1},
-               {0xa9bc, 0xaa29, 109},
-               {0xaa2a, 0xaa2e, 1},
-               {0xaa31, 0xaa32, 1},
-               {0xaa35, 0xaa36, 1},
-               {0xaa43, 0xaa4c, 9},
-               {0xaab0, 0xaab2, 2},
-               {0xaab3, 0xaab4, 1},
-               {0xaab7, 0xaab8, 1},
-               {0xaabe, 0xaabf, 1},
-               {0xaac1, 0xabe5, 292},
-               {0xabe8, 0xabed, 5},
-               {0xfb1e, 0xfe00, 738},
-               {0xfe01, 0xfe0f, 1},
-               {0xfe20, 0xfe26, 1},
-       },
-       R32: []Range32{
-               {0x101fd, 0x10a01, 2052},
-               {0x10a02, 0x10a03, 1},
-               {0x10a05, 0x10a06, 1},
-               {0x10a0c, 0x10a0f, 1},
-               {0x10a38, 0x10a3a, 1},
-               {0x10a3f, 0x11001, 1474},
-               {0x11038, 0x11046, 1},
-               {0x11080, 0x11081, 1},
-               {0x110b3, 0x110b6, 1},
-               {0x110b9, 0x110ba, 1},
-               {0x1d167, 0x1d169, 1},
-               {0x1d17b, 0x1d182, 1},
-               {0x1d185, 0x1d18b, 1},
-               {0x1d1aa, 0x1d1ad, 1},
-               {0x1d242, 0x1d244, 1},
-               {0xe0100, 0xe01ef, 1},
-       },
-}
-
-var _Zl = &RangeTable{
-       R16: []Range16{
-               {0x2028, 0x2028, 1},
-       },
-}
-
-var _Zp = &RangeTable{
-       R16: []Range16{
-               {0x2029, 0x2029, 1},
-       },
-}
-
-var _Zs = &RangeTable{
-       R16: []Range16{
-               {0x0020, 0x00a0, 128},
-               {0x1680, 0x180e, 398},
-               {0x2000, 0x200a, 1},
-               {0x202f, 0x205f, 48},
-               {0x3000, 0x3000, 1},
-       },
-}
-
-var _Cs = &RangeTable{
-       R16: []Range16{
-               {0xd800, 0xdfff, 1},
-       },
-}
-
-var _Co = &RangeTable{
-       R16: []Range16{
-               {0xe000, 0xf8ff, 1},
-       },
-       R32: []Range32{
-               {0xf0000, 0xffffd, 1},
-               {0x100000, 0x10fffd, 1},
-       },
-}
-
-var _Cf = &RangeTable{
-       R16: []Range16{
-               {0x00ad, 0x0600, 1363},
-               {0x0601, 0x0603, 1},
-               {0x06dd, 0x070f, 50},
-               {0x17b4, 0x17b5, 1},
-               {0x200b, 0x200f, 1},
-               {0x202a, 0x202e, 1},
-               {0x2060, 0x2064, 1},
-               {0x206a, 0x206f, 1},
-               {0xfeff, 0xfff9, 250},
-               {0xfffa, 0xfffb, 1},
-       },
-       R32: []Range32{
-               {0x110bd, 0x1d173, 49334},
-               {0x1d174, 0x1d17a, 1},
-               {0xe0001, 0xe0020, 31},
-               {0xe0021, 0xe007f, 1},
-       },
-}
-
-var _Cc = &RangeTable{
-       R16: []Range16{
-               {0x0001, 0x001f, 1},
-               {0x007f, 0x009f, 1},
-       },
-}
-
-var _Po = &RangeTable{
-       R16: []Range16{
-               {0x0021, 0x0023, 1},
-               {0x0025, 0x0027, 1},
-               {0x002a, 0x002e, 2},
-               {0x002f, 0x003a, 11},
-               {0x003b, 0x003f, 4},
-               {0x0040, 0x005c, 28},
-               {0x00a1, 0x00b7, 22},
-               {0x00bf, 0x037e, 703},
-               {0x0387, 0x055a, 467},
-               {0x055b, 0x055f, 1},
-               {0x0589, 0x05c0, 55},
-               {0x05c3, 0x05c6, 3},
-               {0x05f3, 0x05f4, 1},
-               {0x0609, 0x060a, 1},
-               {0x060c, 0x060d, 1},
-               {0x061b, 0x061e, 3},
-               {0x061f, 0x066a, 75},
-               {0x066b, 0x066d, 1},
-               {0x06d4, 0x0700, 44},
-               {0x0701, 0x070d, 1},
-               {0x07f7, 0x07f9, 1},
-               {0x0830, 0x083e, 1},
-               {0x085e, 0x0964, 262},
-               {0x0965, 0x0970, 11},
-               {0x0df4, 0x0e4f, 91},
-               {0x0e5a, 0x0e5b, 1},
-               {0x0f04, 0x0f12, 1},
-               {0x0f85, 0x0fd0, 75},
-               {0x0fd1, 0x0fd4, 1},
-               {0x0fd9, 0x0fda, 1},
-               {0x104a, 0x104f, 1},
-               {0x10fb, 0x1361, 614},
-               {0x1362, 0x1368, 1},
-               {0x166d, 0x166e, 1},
-               {0x16eb, 0x16ed, 1},
-               {0x1735, 0x1736, 1},
-               {0x17d4, 0x17d6, 1},
-               {0x17d8, 0x17da, 1},
-               {0x1800, 0x1805, 1},
-               {0x1807, 0x180a, 1},
-               {0x1944, 0x1945, 1},
-               {0x1a1e, 0x1a1f, 1},
-               {0x1aa0, 0x1aa6, 1},
-               {0x1aa8, 0x1aad, 1},
-               {0x1b5a, 0x1b60, 1},
-               {0x1bfc, 0x1bff, 1},
-               {0x1c3b, 0x1c3f, 1},
-               {0x1c7e, 0x1c7f, 1},
-               {0x1cd3, 0x2016, 835},
-               {0x2017, 0x2020, 9},
-               {0x2021, 0x2027, 1},
-               {0x2030, 0x2038, 1},
-               {0x203b, 0x203e, 1},
-               {0x2041, 0x2043, 1},
-               {0x2047, 0x2051, 1},
-               {0x2053, 0x2055, 2},
-               {0x2056, 0x205e, 1},
-               {0x2cf9, 0x2cfc, 1},
-               {0x2cfe, 0x2cff, 1},
-               {0x2d70, 0x2e00, 144},
-               {0x2e01, 0x2e06, 5},
-               {0x2e07, 0x2e08, 1},
-               {0x2e0b, 0x2e0e, 3},
-               {0x2e0f, 0x2e16, 1},
-               {0x2e18, 0x2e19, 1},
-               {0x2e1b, 0x2e1e, 3},
-               {0x2e1f, 0x2e2a, 11},
-               {0x2e2b, 0x2e2e, 1},
-               {0x2e30, 0x2e31, 1},
-               {0x3001, 0x3003, 1},
-               {0x303d, 0x30fb, 190},
-               {0xa4fe, 0xa4ff, 1},
-               {0xa60d, 0xa60f, 1},
-               {0xa673, 0xa67e, 11},
-               {0xa6f2, 0xa6f7, 1},
-               {0xa874, 0xa877, 1},
-               {0xa8ce, 0xa8cf, 1},
-               {0xa8f8, 0xa8fa, 1},
-               {0xa92e, 0xa92f, 1},
-               {0xa95f, 0xa9c1, 98},
-               {0xa9c2, 0xa9cd, 1},
-               {0xa9de, 0xa9df, 1},
-               {0xaa5c, 0xaa5f, 1},
-               {0xaade, 0xaadf, 1},
-               {0xabeb, 0xfe10, 21029},
-               {0xfe11, 0xfe16, 1},
-               {0xfe19, 0xfe30, 23},
-               {0xfe45, 0xfe46, 1},
-               {0xfe49, 0xfe4c, 1},
-               {0xfe50, 0xfe52, 1},
-               {0xfe54, 0xfe57, 1},
-               {0xfe5f, 0xfe61, 1},
-               {0xfe68, 0xfe6a, 2},
-               {0xfe6b, 0xff01, 150},
-               {0xff02, 0xff03, 1},
-               {0xff05, 0xff07, 1},
-               {0xff0a, 0xff0e, 2},
-               {0xff0f, 0xff1a, 11},
-               {0xff1b, 0xff1f, 4},
-               {0xff20, 0xff3c, 28},
-               {0xff61, 0xff64, 3},
-               {0xff65, 0xff65, 1},
-       },
-       R32: []Range32{
-               {0x10100, 0x10100, 1},
-               {0x10101, 0x1039f, 670},
-               {0x103d0, 0x10857, 1159},
-               {0x1091f, 0x1093f, 32},
-               {0x10a50, 0x10a58, 1},
-               {0x10a7f, 0x10b39, 186},
-               {0x10b3a, 0x10b3f, 1},
-               {0x11047, 0x1104d, 1},
-               {0x110bb, 0x110bc, 1},
-               {0x110be, 0x110c1, 1},
-               {0x12470, 0x12473, 1},
-       },
-}
-
-var _Pi = &RangeTable{
-       R16: []Range16{
-               {0x00ab, 0x2018, 8045},
-               {0x201b, 0x201c, 1},
-               {0x201f, 0x2039, 26},
-               {0x2e02, 0x2e04, 2},
-               {0x2e09, 0x2e0c, 3},
-               {0x2e1c, 0x2e20, 4},
-       },
-}
-
-var _Pf = &RangeTable{
-       R16: []Range16{
-               {0x00bb, 0x2019, 8030},
-               {0x201d, 0x203a, 29},
-               {0x2e03, 0x2e05, 2},
-               {0x2e0a, 0x2e0d, 3},
-               {0x2e1d, 0x2e21, 4},
-       },
-}
-
-var _Pe = &RangeTable{
-       R16: []Range16{
-               {0x0029, 0x005d, 52},
-               {0x007d, 0x0f3b, 3774},
-               {0x0f3d, 0x169c, 1887},
-               {0x2046, 0x207e, 56},
-               {0x208e, 0x232a, 668},
-               {0x2769, 0x2775, 2},
-               {0x27c6, 0x27e7, 33},
-               {0x27e9, 0x27ef, 2},
-               {0x2984, 0x2998, 2},
-               {0x29d9, 0x29db, 2},
-               {0x29fd, 0x2e23, 1062},
-               {0x2e25, 0x2e29, 2},
-               {0x3009, 0x3011, 2},
-               {0x3015, 0x301b, 2},
-               {0x301e, 0x301f, 1},
-               {0xfd3f, 0xfe18, 217},
-               {0xfe36, 0xfe44, 2},
-               {0xfe48, 0xfe5a, 18},
-               {0xfe5c, 0xfe5e, 2},
-               {0xff09, 0xff3d, 52},
-               {0xff5d, 0xff63, 3},
-       },
-}
-
-var _Pd = &RangeTable{
-       R16: []Range16{
-               {0x002d, 0x058a, 1373},
-               {0x05be, 0x1400, 3650},
-               {0x1806, 0x2010, 2058},
-               {0x2011, 0x2015, 1},
-               {0x2e17, 0x2e1a, 3},
-               {0x301c, 0x3030, 20},
-               {0x30a0, 0xfe31, 52625},
-               {0xfe32, 0xfe58, 38},
-               {0xfe63, 0xff0d, 170},
-       },
-}
-
-var _Pc = &RangeTable{
-       R16: []Range16{
-               {0x005f, 0x203f, 8160},
-               {0x2040, 0x2054, 20},
-               {0xfe33, 0xfe34, 1},
-               {0xfe4d, 0xfe4f, 1},
-               {0xff3f, 0xff3f, 1},
-       },
-}
-
-var _Ps = &RangeTable{
-       R16: []Range16{
-               {0x0028, 0x005b, 51},
-               {0x007b, 0x0f3a, 3775},
-               {0x0f3c, 0x169b, 1887},
-               {0x201a, 0x201e, 4},
-               {0x2045, 0x207d, 56},
-               {0x208d, 0x2329, 668},
-               {0x2768, 0x2774, 2},
-               {0x27c5, 0x27e6, 33},
-               {0x27e8, 0x27ee, 2},
-               {0x2983, 0x2997, 2},
-               {0x29d8, 0x29da, 2},
-               {0x29fc, 0x2e22, 1062},
-               {0x2e24, 0x2e28, 2},
-               {0x3008, 0x3010, 2},
-               {0x3014, 0x301a, 2},
-               {0x301d, 0xfd3e, 52513},
-               {0xfe17, 0xfe35, 30},
-               {0xfe37, 0xfe43, 2},
-               {0xfe47, 0xfe59, 18},
-               {0xfe5b, 0xfe5d, 2},
-               {0xff08, 0xff3b, 51},
-               {0xff5b, 0xff5f, 4},
-               {0xff62, 0xff62, 1},
-       },
-}
-
-var _Nd = &RangeTable{
-       R16: []Range16{
-               {0x0030, 0x0039, 1},
-               {0x0660, 0x0669, 1},
-               {0x06f0, 0x06f9, 1},
-               {0x07c0, 0x07c9, 1},
-               {0x0966, 0x096f, 1},
-               {0x09e6, 0x09ef, 1},
-               {0x0a66, 0x0a6f, 1},
-               {0x0ae6, 0x0aef, 1},
-               {0x0b66, 0x0b6f, 1},
-               {0x0be6, 0x0bef, 1},
-               {0x0c66, 0x0c6f, 1},
-               {0x0ce6, 0x0cef, 1},
-               {0x0d66, 0x0d6f, 1},
-               {0x0e50, 0x0e59, 1},
-               {0x0ed0, 0x0ed9, 1},
-               {0x0f20, 0x0f29, 1},
-               {0x1040, 0x1049, 1},
-               {0x1090, 0x1099, 1},
-               {0x17e0, 0x17e9, 1},
-               {0x1810, 0x1819, 1},
-               {0x1946, 0x194f, 1},
-               {0x19d0, 0x19d9, 1},
-               {0x1a80, 0x1a89, 1},
-               {0x1a90, 0x1a99, 1},
-               {0x1b50, 0x1b59, 1},
-               {0x1bb0, 0x1bb9, 1},
-               {0x1c40, 0x1c49, 1},
-               {0x1c50, 0x1c59, 1},
-               {0xa620, 0xa629, 1},
-               {0xa8d0, 0xa8d9, 1},
-               {0xa900, 0xa909, 1},
-               {0xa9d0, 0xa9d9, 1},
-               {0xaa50, 0xaa59, 1},
-               {0xabf0, 0xabf9, 1},
-               {0xff10, 0xff19, 1},
-       },
-       R32: []Range32{
-               {0x104a0, 0x104a9, 1},
-               {0x11066, 0x1106f, 1},
-               {0x1d7ce, 0x1d7ff, 1},
-       },
-}
-
-var _Nl = &RangeTable{
-       R16: []Range16{
-               {0x16ee, 0x16f0, 1},
-               {0x2160, 0x2182, 1},
-               {0x2185, 0x2188, 1},
-               {0x3007, 0x3021, 26},
-               {0x3022, 0x3029, 1},
-               {0x3038, 0x303a, 1},
-               {0xa6e6, 0xa6ef, 1},
-       },
-       R32: []Range32{
-               {0x10140, 0x10174, 1},
-               {0x10341, 0x1034a, 9},
-               {0x103d1, 0x103d5, 1},
-               {0x12400, 0x12462, 1},
-       },
-}
-
-var _No = &RangeTable{
-       R16: []Range16{
-               {0x00b2, 0x00b3, 1},
-               {0x00b9, 0x00bc, 3},
-               {0x00bd, 0x00be, 1},
-               {0x09f4, 0x09f9, 1},
-               {0x0b72, 0x0b77, 1},
-               {0x0bf0, 0x0bf2, 1},
-               {0x0c78, 0x0c7e, 1},
-               {0x0d70, 0x0d75, 1},
-               {0x0f2a, 0x0f33, 1},
-               {0x1369, 0x137c, 1},
-               {0x17f0, 0x17f9, 1},
-               {0x19da, 0x2070, 1686},
-               {0x2074, 0x2079, 1},
-               {0x2080, 0x2089, 1},
-               {0x2150, 0x215f, 1},
-               {0x2189, 0x2460, 727},
-               {0x2461, 0x249b, 1},
-               {0x24ea, 0x24ff, 1},
-               {0x2776, 0x2793, 1},
-               {0x2cfd, 0x3192, 1173},
-               {0x3193, 0x3195, 1},
-               {0x3220, 0x3229, 1},
-               {0x3251, 0x325f, 1},
-               {0x3280, 0x3289, 1},
-               {0x32b1, 0x32bf, 1},
-               {0xa830, 0xa835, 1},
-       },
-       R32: []Range32{
-               {0x10107, 0x10133, 1},
-               {0x10175, 0x10178, 1},
-               {0x1018a, 0x10320, 406},
-               {0x10321, 0x10323, 1},
-               {0x10858, 0x1085f, 1},
-               {0x10916, 0x1091b, 1},
-               {0x10a40, 0x10a47, 1},
-               {0x10a7d, 0x10a7e, 1},
-               {0x10b58, 0x10b5f, 1},
-               {0x10b78, 0x10b7f, 1},
-               {0x10e60, 0x10e7e, 1},
-               {0x11052, 0x11065, 1},
-               {0x1d360, 0x1d371, 1},
-               {0x1f100, 0x1f10a, 1},
-       },
-}
-
-var _So = &RangeTable{
-       R16: []Range16{
-               {0x00a6, 0x00a7, 1},
-               {0x00a9, 0x00ae, 5},
-               {0x00b0, 0x00b6, 6},
-               {0x0482, 0x060e, 396},
-               {0x060f, 0x06de, 207},
-               {0x06e9, 0x06fd, 20},
-               {0x06fe, 0x07f6, 248},
-               {0x09fa, 0x0b70, 374},
-               {0x0bf3, 0x0bf8, 1},
-               {0x0bfa, 0x0c7f, 133},
-               {0x0d79, 0x0f01, 392},
-               {0x0f02, 0x0f03, 1},
-               {0x0f13, 0x0f17, 1},
-               {0x0f1a, 0x0f1f, 1},
-               {0x0f34, 0x0f38, 2},
-               {0x0fbe, 0x0fc5, 1},
-               {0x0fc7, 0x0fcc, 1},
-               {0x0fce, 0x0fcf, 1},
-               {0x0fd5, 0x0fd8, 1},
-               {0x109e, 0x109f, 1},
-               {0x1360, 0x1390, 48},
-               {0x1391, 0x1399, 1},
-               {0x1940, 0x19de, 158},
-               {0x19df, 0x19ff, 1},
-               {0x1b61, 0x1b6a, 1},
-               {0x1b74, 0x1b7c, 1},
-               {0x2100, 0x2101, 1},
-               {0x2103, 0x2106, 1},
-               {0x2108, 0x2109, 1},
-               {0x2114, 0x2116, 2},
-               {0x2117, 0x211e, 7},
-               {0x211f, 0x2123, 1},
-               {0x2125, 0x2129, 2},
-               {0x212e, 0x213a, 12},
-               {0x213b, 0x214a, 15},
-               {0x214c, 0x214d, 1},
-               {0x214f, 0x2195, 70},
-               {0x2196, 0x2199, 1},
-               {0x219c, 0x219f, 1},
-               {0x21a1, 0x21a2, 1},
-               {0x21a4, 0x21a5, 1},
-               {0x21a7, 0x21ad, 1},
-               {0x21af, 0x21cd, 1},
-               {0x21d0, 0x21d1, 1},
-               {0x21d3, 0x21d5, 2},
-               {0x21d6, 0x21f3, 1},
-               {0x2300, 0x2307, 1},
-               {0x230c, 0x231f, 1},
-               {0x2322, 0x2328, 1},
-               {0x232b, 0x237b, 1},
-               {0x237d, 0x239a, 1},
-               {0x23b4, 0x23db, 1},
-               {0x23e2, 0x23f3, 1},
-               {0x2400, 0x2426, 1},
-               {0x2440, 0x244a, 1},
-               {0x249c, 0x24e9, 1},
-               {0x2500, 0x25b6, 1},
-               {0x25b8, 0x25c0, 1},
-               {0x25c2, 0x25f7, 1},
-               {0x2600, 0x266e, 1},
-               {0x2670, 0x26ff, 1},
-               {0x2701, 0x2767, 1},
-               {0x2794, 0x27bf, 1},
-               {0x2800, 0x28ff, 1},
-               {0x2b00, 0x2b2f, 1},
-               {0x2b45, 0x2b46, 1},
-               {0x2b50, 0x2b59, 1},
-               {0x2ce5, 0x2cea, 1},
-               {0x2e80, 0x2e99, 1},
-               {0x2e9b, 0x2ef3, 1},
-               {0x2f00, 0x2fd5, 1},
-               {0x2ff0, 0x2ffb, 1},
-               {0x3004, 0x3012, 14},
-               {0x3013, 0x3020, 13},
-               {0x3036, 0x3037, 1},
-               {0x303e, 0x303f, 1},
-               {0x3190, 0x3191, 1},
-               {0x3196, 0x319f, 1},
-               {0x31c0, 0x31e3, 1},
-               {0x3200, 0x321e, 1},
-               {0x322a, 0x3250, 1},
-               {0x3260, 0x327f, 1},
-               {0x328a, 0x32b0, 1},
-               {0x32c0, 0x32fe, 1},
-               {0x3300, 0x33ff, 1},
-               {0x4dc0, 0x4dff, 1},
-               {0xa490, 0xa4c6, 1},
-               {0xa828, 0xa82b, 1},
-               {0xa836, 0xa837, 1},
-               {0xa839, 0xaa77, 574},
-               {0xaa78, 0xaa79, 1},
-               {0xfdfd, 0xffe4, 487},
-               {0xffe8, 0xffed, 5},
-               {0xffee, 0xfffc, 14},
-               {0xfffd, 0xfffd, 1},
-       },
-       R32: []Range32{
-               {0x10102, 0x10102, 1},
-               {0x10137, 0x1013f, 1},
-               {0x10179, 0x10189, 1},
-               {0x10190, 0x1019b, 1},
-               {0x101d0, 0x101fc, 1},
-               {0x1d000, 0x1d0f5, 1},
-               {0x1d100, 0x1d126, 1},
-               {0x1d129, 0x1d164, 1},
-               {0x1d16a, 0x1d16c, 1},
-               {0x1d183, 0x1d184, 1},
-               {0x1d18c, 0x1d1a9, 1},
-               {0x1d1ae, 0x1d1dd, 1},
-               {0x1d200, 0x1d241, 1},
-               {0x1d245, 0x1d300, 187},
-               {0x1d301, 0x1d356, 1},
-               {0x1f000, 0x1f02b, 1},
-               {0x1f030, 0x1f093, 1},
-               {0x1f0a0, 0x1f0ae, 1},
-               {0x1f0b1, 0x1f0be, 1},
-               {0x1f0c1, 0x1f0cf, 1},
-               {0x1f0d1, 0x1f0df, 1},
-               {0x1f110, 0x1f12e, 1},
-               {0x1f130, 0x1f169, 1},
-               {0x1f170, 0x1f19a, 1},
-               {0x1f1e6, 0x1f202, 1},
-               {0x1f210, 0x1f23a, 1},
-               {0x1f240, 0x1f248, 1},
-               {0x1f250, 0x1f251, 1},
-               {0x1f300, 0x1f320, 1},
-               {0x1f330, 0x1f335, 1},
-               {0x1f337, 0x1f37c, 1},
-               {0x1f380, 0x1f393, 1},
-               {0x1f3a0, 0x1f3c4, 1},
-               {0x1f3c6, 0x1f3ca, 1},
-               {0x1f3e0, 0x1f3f0, 1},
-               {0x1f400, 0x1f43e, 1},
-               {0x1f440, 0x1f442, 2},
-               {0x1f443, 0x1f4f7, 1},
-               {0x1f4f9, 0x1f4fc, 1},
-               {0x1f500, 0x1f53d, 1},
-               {0x1f550, 0x1f567, 1},
-               {0x1f5fb, 0x1f5ff, 1},
-               {0x1f601, 0x1f610, 1},
-               {0x1f612, 0x1f614, 1},
-               {0x1f616, 0x1f61c, 2},
-               {0x1f61d, 0x1f61e, 1},
-               {0x1f620, 0x1f625, 1},
-               {0x1f628, 0x1f62b, 1},
-               {0x1f62d, 0x1f630, 3},
-               {0x1f631, 0x1f633, 1},
-               {0x1f635, 0x1f640, 1},
-               {0x1f645, 0x1f64f, 1},
-               {0x1f680, 0x1f6c5, 1},
-               {0x1f700, 0x1f773, 1},
-       },
-}
-
-var _Sm = &RangeTable{
-       R16: []Range16{
-               {0x002b, 0x003c, 17},
-               {0x003d, 0x003e, 1},
-               {0x007c, 0x007e, 2},
-               {0x00ac, 0x00b1, 5},
-               {0x00d7, 0x00f7, 32},
-               {0x03f6, 0x0606, 528},
-               {0x0607, 0x0608, 1},
-               {0x2044, 0x2052, 14},
-               {0x207a, 0x207c, 1},
-               {0x208a, 0x208c, 1},
-               {0x2118, 0x2140, 40},
-               {0x2141, 0x2144, 1},
-               {0x214b, 0x2190, 69},
-               {0x2191, 0x2194, 1},
-               {0x219a, 0x219b, 1},
-               {0x21a0, 0x21a6, 3},
-               {0x21ae, 0x21ce, 32},
-               {0x21cf, 0x21d2, 3},
-               {0x21d4, 0x21f4, 32},
-               {0x21f5, 0x22ff, 1},
-               {0x2308, 0x230b, 1},
-               {0x2320, 0x2321, 1},
-               {0x237c, 0x239b, 31},
-               {0x239c, 0x23b3, 1},
-               {0x23dc, 0x23e1, 1},
-               {0x25b7, 0x25c1, 10},
-               {0x25f8, 0x25ff, 1},
-               {0x266f, 0x27c0, 337},
-               {0x27c1, 0x27c4, 1},
-               {0x27c7, 0x27ca, 1},
-               {0x27cc, 0x27ce, 2},
-               {0x27cf, 0x27e5, 1},
-               {0x27f0, 0x27ff, 1},
-               {0x2900, 0x2982, 1},
-               {0x2999, 0x29d7, 1},
-               {0x29dc, 0x29fb, 1},
-               {0x29fe, 0x2aff, 1},
-               {0x2b30, 0x2b44, 1},
-               {0x2b47, 0x2b4c, 1},
-               {0xfb29, 0xfe62, 825},
-               {0xfe64, 0xfe66, 1},
-               {0xff0b, 0xff1c, 17},
-               {0xff1d, 0xff1e, 1},
-               {0xff5c, 0xff5e, 2},
-               {0xffe2, 0xffe9, 7},
-               {0xffea, 0xffec, 1},
-       },
-       R32: []Range32{
-               {0x1d6c1, 0x1d6db, 26},
-               {0x1d6fb, 0x1d715, 26},
-               {0x1d735, 0x1d74f, 26},
-               {0x1d76f, 0x1d789, 26},
-               {0x1d7a9, 0x1d7c3, 26},
-       },
-}
-
-var _Sk = &RangeTable{
-       R16: []Range16{
-               {0x005e, 0x0060, 2},
-               {0x00a8, 0x00af, 7},
-               {0x00b4, 0x00b8, 4},
-               {0x02c2, 0x02c5, 1},
-               {0x02d2, 0x02df, 1},
-               {0x02e5, 0x02eb, 1},
-               {0x02ed, 0x02ef, 2},
-               {0x02f0, 0x02ff, 1},
-               {0x0375, 0x0384, 15},
-               {0x0385, 0x1fbd, 7224},
-               {0x1fbf, 0x1fc1, 1},
-               {0x1fcd, 0x1fcf, 1},
-               {0x1fdd, 0x1fdf, 1},
-               {0x1fed, 0x1fef, 1},
-               {0x1ffd, 0x1ffe, 1},
-               {0x309b, 0x309c, 1},
-               {0xa700, 0xa716, 1},
-               {0xa720, 0xa721, 1},
-               {0xa789, 0xa78a, 1},
-               {0xfbb2, 0xfbc1, 1},
-               {0xff3e, 0xff40, 2},
-               {0xffe3, 0xffe3, 1},
-       },
-}
-
-var _Sc = &RangeTable{
-       R16: []Range16{
-               {0x0024, 0x00a2, 126},
-               {0x00a3, 0x00a5, 1},
-               {0x060b, 0x09f2, 999},
-               {0x09f3, 0x09fb, 8},
-               {0x0af1, 0x0bf9, 264},
-               {0x0e3f, 0x17db, 2460},
-               {0x20a0, 0x20b9, 1},
-               {0xa838, 0xfdfc, 21956},
-               {0xfe69, 0xff04, 155},
-               {0xffe0, 0xffe1, 1},
-               {0xffe5, 0xffe6, 1},
-       },
-}
-
-var _Lu = &RangeTable{
-       R16: []Range16{
-               {0x0041, 0x005a, 1},
-               {0x00c0, 0x00d6, 1},
-               {0x00d8, 0x00de, 1},
-               {0x0100, 0x0136, 2},
-               {0x0139, 0x0147, 2},
-               {0x014a, 0x0178, 2},
-               {0x0179, 0x017d, 2},
-               {0x0181, 0x0182, 1},
-               {0x0184, 0x0186, 2},
-               {0x0187, 0x0189, 2},
-               {0x018a, 0x018b, 1},
-               {0x018e, 0x0191, 1},
-               {0x0193, 0x0194, 1},
-               {0x0196, 0x0198, 1},
-               {0x019c, 0x019d, 1},
-               {0x019f, 0x01a0, 1},
-               {0x01a2, 0x01a6, 2},
-               {0x01a7, 0x01a9, 2},
-               {0x01ac, 0x01ae, 2},
-               {0x01af, 0x01b1, 2},
-               {0x01b2, 0x01b3, 1},
-               {0x01b5, 0x01b7, 2},
-               {0x01b8, 0x01bc, 4},
-               {0x01c4, 0x01cd, 3},
-               {0x01cf, 0x01db, 2},
-               {0x01de, 0x01ee, 2},
-               {0x01f1, 0x01f4, 3},
-               {0x01f6, 0x01f8, 1},
-               {0x01fa, 0x0232, 2},
-               {0x023a, 0x023b, 1},
-               {0x023d, 0x023e, 1},
-               {0x0241, 0x0243, 2},
-               {0x0244, 0x0246, 1},
-               {0x0248, 0x024e, 2},
-               {0x0370, 0x0372, 2},
-               {0x0376, 0x0386, 16},
-               {0x0388, 0x038a, 1},
-               {0x038c, 0x038e, 2},
-               {0x038f, 0x0391, 2},
-               {0x0392, 0x03a1, 1},
-               {0x03a3, 0x03ab, 1},
-               {0x03cf, 0x03d2, 3},
-               {0x03d3, 0x03d4, 1},
-               {0x03d8, 0x03ee, 2},
-               {0x03f4, 0x03f7, 3},
-               {0x03f9, 0x03fa, 1},
-               {0x03fd, 0x042f, 1},
-               {0x0460, 0x0480, 2},
-               {0x048a, 0x04c0, 2},
-               {0x04c1, 0x04cd, 2},
-               {0x04d0, 0x0526, 2},
-               {0x0531, 0x0556, 1},
-               {0x10a0, 0x10c5, 1},
-               {0x1e00, 0x1e94, 2},
-               {0x1e9e, 0x1efe, 2},
-               {0x1f08, 0x1f0f, 1},
-               {0x1f18, 0x1f1d, 1},
-               {0x1f28, 0x1f2f, 1},
-               {0x1f38, 0x1f3f, 1},
-               {0x1f48, 0x1f4d, 1},
-               {0x1f59, 0x1f5f, 2},
-               {0x1f68, 0x1f6f, 1},
-               {0x1fb8, 0x1fbb, 1},
-               {0x1fc8, 0x1fcb, 1},
-               {0x1fd8, 0x1fdb, 1},
-               {0x1fe8, 0x1fec, 1},
-               {0x1ff8, 0x1ffb, 1},
-               {0x2102, 0x2107, 5},
-               {0x210b, 0x210d, 1},
-               {0x2110, 0x2112, 1},
-               {0x2115, 0x2119, 4},
-               {0x211a, 0x211d, 1},
-               {0x2124, 0x212a, 2},
-               {0x212b, 0x212d, 1},
-               {0x2130, 0x2133, 1},
-               {0x213e, 0x213f, 1},
-               {0x2145, 0x2183, 62},
-               {0x2c00, 0x2c2e, 1},
-               {0x2c60, 0x2c62, 2},
-               {0x2c63, 0x2c64, 1},
-               {0x2c67, 0x2c6d, 2},
-               {0x2c6e, 0x2c70, 1},
-               {0x2c72, 0x2c75, 3},
-               {0x2c7e, 0x2c80, 1},
-               {0x2c82, 0x2ce2, 2},
-               {0x2ceb, 0x2ced, 2},
-               {0xa640, 0xa66c, 2},
-               {0xa680, 0xa696, 2},
-               {0xa722, 0xa72e, 2},
-               {0xa732, 0xa76e, 2},
-               {0xa779, 0xa77d, 2},
-               {0xa77e, 0xa786, 2},
-               {0xa78b, 0xa78d, 2},
-               {0xa790, 0xa7a0, 16},
-               {0xa7a2, 0xa7a8, 2},
-               {0xff21, 0xff3a, 1},
-       },
-       R32: []Range32{
-               {0x10400, 0x10427, 1},
-               {0x1d400, 0x1d419, 1},
-               {0x1d434, 0x1d44d, 1},
-               {0x1d468, 0x1d481, 1},
-               {0x1d49c, 0x1d49e, 2},
-               {0x1d49f, 0x1d4a5, 3},
-               {0x1d4a6, 0x1d4a9, 3},
-               {0x1d4aa, 0x1d4ac, 1},
-               {0x1d4ae, 0x1d4b5, 1},
-               {0x1d4d0, 0x1d4e9, 1},
-               {0x1d504, 0x1d505, 1},
-               {0x1d507, 0x1d50a, 1},
-               {0x1d50d, 0x1d514, 1},
-               {0x1d516, 0x1d51c, 1},
-               {0x1d538, 0x1d539, 1},
-               {0x1d53b, 0x1d53e, 1},
-               {0x1d540, 0x1d544, 1},
-               {0x1d546, 0x1d54a, 4},
-               {0x1d54b, 0x1d550, 1},
-               {0x1d56c, 0x1d585, 1},
-               {0x1d5a0, 0x1d5b9, 1},
-               {0x1d5d4, 0x1d5ed, 1},
-               {0x1d608, 0x1d621, 1},
-               {0x1d63c, 0x1d655, 1},
-               {0x1d670, 0x1d689, 1},
-               {0x1d6a8, 0x1d6c0, 1},
-               {0x1d6e2, 0x1d6fa, 1},
-               {0x1d71c, 0x1d734, 1},
-               {0x1d756, 0x1d76e, 1},
-               {0x1d790, 0x1d7a8, 1},
-               {0x1d7ca, 0x1d7ca, 1},
-       },
-}
-
-var _Lt = &RangeTable{
-       R16: []Range16{
-               {0x01c5, 0x01cb, 3},
-               {0x01f2, 0x1f88, 7574},
-               {0x1f89, 0x1f8f, 1},
-               {0x1f98, 0x1f9f, 1},
-               {0x1fa8, 0x1faf, 1},
-               {0x1fbc, 0x1fcc, 16},
-               {0x1ffc, 0x1ffc, 1},
-       },
-}
-
-var _Lo = &RangeTable{
+var _Lo = &RangeTable{
        R16: []Range16{
                {0x01bb, 0x01c0, 5},
                {0x01c1, 0x01c3, 1},
@@ -2655,49 +996,1708 @@ var _Lo = &RangeTable{
                {0xffda, 0xffdc, 1},
        },
        R32: []Range32{
-               {0x10000, 0x1000b, 1},
-               {0x1000d, 0x10026, 1},
-               {0x10028, 0x1003a, 1},
-               {0x1003c, 0x1003d, 1},
-               {0x1003f, 0x1004d, 1},
-               {0x10050, 0x1005d, 1},
-               {0x10080, 0x100fa, 1},
-               {0x10280, 0x1029c, 1},
-               {0x102a0, 0x102d0, 1},
-               {0x10300, 0x1031e, 1},
-               {0x10330, 0x10340, 1},
-               {0x10342, 0x10349, 1},
-               {0x10380, 0x1039d, 1},
-               {0x103a0, 0x103c3, 1},
-               {0x103c8, 0x103cf, 1},
-               {0x10450, 0x1049d, 1},
-               {0x10800, 0x10805, 1},
-               {0x10808, 0x1080a, 2},
-               {0x1080b, 0x10835, 1},
-               {0x10837, 0x10838, 1},
-               {0x1083c, 0x1083f, 3},
-               {0x10840, 0x10855, 1},
-               {0x10900, 0x10915, 1},
-               {0x10920, 0x10939, 1},
-               {0x10a00, 0x10a10, 16},
-               {0x10a11, 0x10a13, 1},
-               {0x10a15, 0x10a17, 1},
-               {0x10a19, 0x10a33, 1},
-               {0x10a60, 0x10a7c, 1},
-               {0x10b00, 0x10b35, 1},
-               {0x10b40, 0x10b55, 1},
-               {0x10b60, 0x10b72, 1},
-               {0x10c00, 0x10c48, 1},
-               {0x11003, 0x11037, 1},
-               {0x11083, 0x110af, 1},
-               {0x12000, 0x1236e, 1},
-               {0x13000, 0x1342e, 1},
-               {0x16800, 0x16a38, 1},
-               {0x1b000, 0x1b001, 1},
-               {0x20000, 0x2a6d6, 1},
-               {0x2a700, 0x2b734, 1},
-               {0x2b740, 0x2b81d, 1},
-               {0x2f800, 0x2fa1d, 1},
+               {0x10000, 0x1000b, 1},
+               {0x1000d, 0x10026, 1},
+               {0x10028, 0x1003a, 1},
+               {0x1003c, 0x1003d, 1},
+               {0x1003f, 0x1004d, 1},
+               {0x10050, 0x1005d, 1},
+               {0x10080, 0x100fa, 1},
+               {0x10280, 0x1029c, 1},
+               {0x102a0, 0x102d0, 1},
+               {0x10300, 0x1031e, 1},
+               {0x10330, 0x10340, 1},
+               {0x10342, 0x10349, 1},
+               {0x10380, 0x1039d, 1},
+               {0x103a0, 0x103c3, 1},
+               {0x103c8, 0x103cf, 1},
+               {0x10450, 0x1049d, 1},
+               {0x10800, 0x10805, 1},
+               {0x10808, 0x1080a, 2},
+               {0x1080b, 0x10835, 1},
+               {0x10837, 0x10838, 1},
+               {0x1083c, 0x1083f, 3},
+               {0x10840, 0x10855, 1},
+               {0x10900, 0x10915, 1},
+               {0x10920, 0x10939, 1},
+               {0x10a00, 0x10a10, 16},
+               {0x10a11, 0x10a13, 1},
+               {0x10a15, 0x10a17, 1},
+               {0x10a19, 0x10a33, 1},
+               {0x10a60, 0x10a7c, 1},
+               {0x10b00, 0x10b35, 1},
+               {0x10b40, 0x10b55, 1},
+               {0x10b60, 0x10b72, 1},
+               {0x10c00, 0x10c48, 1},
+               {0x11003, 0x11037, 1},
+               {0x11083, 0x110af, 1},
+               {0x12000, 0x1236e, 1},
+               {0x13000, 0x1342e, 1},
+               {0x16800, 0x16a38, 1},
+               {0x1b000, 0x1b001, 1},
+               {0x20000, 0x2a6d6, 1},
+               {0x2a700, 0x2b734, 1},
+               {0x2b740, 0x2b81d, 1},
+               {0x2f800, 0x2fa1d, 1},
+       },
+}
+
+var _Lt = &RangeTable{
+       R16: []Range16{
+               {0x01c5, 0x01cb, 3},
+               {0x01f2, 0x1f88, 7574},
+               {0x1f89, 0x1f8f, 1},
+               {0x1f98, 0x1f9f, 1},
+               {0x1fa8, 0x1faf, 1},
+               {0x1fbc, 0x1fcc, 16},
+               {0x1ffc, 0x1ffc, 1},
+       },
+}
+
+var _Lu = &RangeTable{
+       R16: []Range16{
+               {0x0041, 0x005a, 1},
+               {0x00c0, 0x00d6, 1},
+               {0x00d8, 0x00de, 1},
+               {0x0100, 0x0136, 2},
+               {0x0139, 0x0147, 2},
+               {0x014a, 0x0178, 2},
+               {0x0179, 0x017d, 2},
+               {0x0181, 0x0182, 1},
+               {0x0184, 0x0186, 2},
+               {0x0187, 0x0189, 2},
+               {0x018a, 0x018b, 1},
+               {0x018e, 0x0191, 1},
+               {0x0193, 0x0194, 1},
+               {0x0196, 0x0198, 1},
+               {0x019c, 0x019d, 1},
+               {0x019f, 0x01a0, 1},
+               {0x01a2, 0x01a6, 2},
+               {0x01a7, 0x01a9, 2},
+               {0x01ac, 0x01ae, 2},
+               {0x01af, 0x01b1, 2},
+               {0x01b2, 0x01b3, 1},
+               {0x01b5, 0x01b7, 2},
+               {0x01b8, 0x01bc, 4},
+               {0x01c4, 0x01cd, 3},
+               {0x01cf, 0x01db, 2},
+               {0x01de, 0x01ee, 2},
+               {0x01f1, 0x01f4, 3},
+               {0x01f6, 0x01f8, 1},
+               {0x01fa, 0x0232, 2},
+               {0x023a, 0x023b, 1},
+               {0x023d, 0x023e, 1},
+               {0x0241, 0x0243, 2},
+               {0x0244, 0x0246, 1},
+               {0x0248, 0x024e, 2},
+               {0x0370, 0x0372, 2},
+               {0x0376, 0x0386, 16},
+               {0x0388, 0x038a, 1},
+               {0x038c, 0x038e, 2},
+               {0x038f, 0x0391, 2},
+               {0x0392, 0x03a1, 1},
+               {0x03a3, 0x03ab, 1},
+               {0x03cf, 0x03d2, 3},
+               {0x03d3, 0x03d4, 1},
+               {0x03d8, 0x03ee, 2},
+               {0x03f4, 0x03f7, 3},
+               {0x03f9, 0x03fa, 1},
+               {0x03fd, 0x042f, 1},
+               {0x0460, 0x0480, 2},
+               {0x048a, 0x04c0, 2},
+               {0x04c1, 0x04cd, 2},
+               {0x04d0, 0x0526, 2},
+               {0x0531, 0x0556, 1},
+               {0x10a0, 0x10c5, 1},
+               {0x1e00, 0x1e94, 2},
+               {0x1e9e, 0x1efe, 2},
+               {0x1f08, 0x1f0f, 1},
+               {0x1f18, 0x1f1d, 1},
+               {0x1f28, 0x1f2f, 1},
+               {0x1f38, 0x1f3f, 1},
+               {0x1f48, 0x1f4d, 1},
+               {0x1f59, 0x1f5f, 2},
+               {0x1f68, 0x1f6f, 1},
+               {0x1fb8, 0x1fbb, 1},
+               {0x1fc8, 0x1fcb, 1},
+               {0x1fd8, 0x1fdb, 1},
+               {0x1fe8, 0x1fec, 1},
+               {0x1ff8, 0x1ffb, 1},
+               {0x2102, 0x2107, 5},
+               {0x210b, 0x210d, 1},
+               {0x2110, 0x2112, 1},
+               {0x2115, 0x2119, 4},
+               {0x211a, 0x211d, 1},
+               {0x2124, 0x212a, 2},
+               {0x212b, 0x212d, 1},
+               {0x2130, 0x2133, 1},
+               {0x213e, 0x213f, 1},
+               {0x2145, 0x2183, 62},
+               {0x2c00, 0x2c2e, 1},
+               {0x2c60, 0x2c62, 2},
+               {0x2c63, 0x2c64, 1},
+               {0x2c67, 0x2c6d, 2},
+               {0x2c6e, 0x2c70, 1},
+               {0x2c72, 0x2c75, 3},
+               {0x2c7e, 0x2c80, 1},
+               {0x2c82, 0x2ce2, 2},
+               {0x2ceb, 0x2ced, 2},
+               {0xa640, 0xa66c, 2},
+               {0xa680, 0xa696, 2},
+               {0xa722, 0xa72e, 2},
+               {0xa732, 0xa76e, 2},
+               {0xa779, 0xa77d, 2},
+               {0xa77e, 0xa786, 2},
+               {0xa78b, 0xa78d, 2},
+               {0xa790, 0xa7a0, 16},
+               {0xa7a2, 0xa7a8, 2},
+               {0xff21, 0xff3a, 1},
+       },
+       R32: []Range32{
+               {0x10400, 0x10427, 1},
+               {0x1d400, 0x1d419, 1},
+               {0x1d434, 0x1d44d, 1},
+               {0x1d468, 0x1d481, 1},
+               {0x1d49c, 0x1d49e, 2},
+               {0x1d49f, 0x1d4a5, 3},
+               {0x1d4a6, 0x1d4a9, 3},
+               {0x1d4aa, 0x1d4ac, 1},
+               {0x1d4ae, 0x1d4b5, 1},
+               {0x1d4d0, 0x1d4e9, 1},
+               {0x1d504, 0x1d505, 1},
+               {0x1d507, 0x1d50a, 1},
+               {0x1d50d, 0x1d514, 1},
+               {0x1d516, 0x1d51c, 1},
+               {0x1d538, 0x1d539, 1},
+               {0x1d53b, 0x1d53e, 1},
+               {0x1d540, 0x1d544, 1},
+               {0x1d546, 0x1d54a, 4},
+               {0x1d54b, 0x1d550, 1},
+               {0x1d56c, 0x1d585, 1},
+               {0x1d5a0, 0x1d5b9, 1},
+               {0x1d5d4, 0x1d5ed, 1},
+               {0x1d608, 0x1d621, 1},
+               {0x1d63c, 0x1d655, 1},
+               {0x1d670, 0x1d689, 1},
+               {0x1d6a8, 0x1d6c0, 1},
+               {0x1d6e2, 0x1d6fa, 1},
+               {0x1d71c, 0x1d734, 1},
+               {0x1d756, 0x1d76e, 1},
+               {0x1d790, 0x1d7a8, 1},
+               {0x1d7ca, 0x1d7ca, 1},
+       },
+}
+
+var _M = &RangeTable{
+       R16: []Range16{
+               {0x0300, 0x036f, 1},
+               {0x0483, 0x0489, 1},
+               {0x0591, 0x05bd, 1},
+               {0x05bf, 0x05c1, 2},
+               {0x05c2, 0x05c4, 2},
+               {0x05c5, 0x05c7, 2},
+               {0x0610, 0x061a, 1},
+               {0x064b, 0x065f, 1},
+               {0x0670, 0x06d6, 102},
+               {0x06d7, 0x06dc, 1},
+               {0x06df, 0x06e4, 1},
+               {0x06e7, 0x06e8, 1},
+               {0x06ea, 0x06ed, 1},
+               {0x0711, 0x0730, 31},
+               {0x0731, 0x074a, 1},
+               {0x07a6, 0x07b0, 1},
+               {0x07eb, 0x07f3, 1},
+               {0x0816, 0x0819, 1},
+               {0x081b, 0x0823, 1},
+               {0x0825, 0x0827, 1},
+               {0x0829, 0x082d, 1},
+               {0x0859, 0x085b, 1},
+               {0x0900, 0x0903, 1},
+               {0x093a, 0x093c, 1},
+               {0x093e, 0x094f, 1},
+               {0x0951, 0x0957, 1},
+               {0x0962, 0x0963, 1},
+               {0x0981, 0x0983, 1},
+               {0x09bc, 0x09be, 2},
+               {0x09bf, 0x09c4, 1},
+               {0x09c7, 0x09c8, 1},
+               {0x09cb, 0x09cd, 1},
+               {0x09d7, 0x09e2, 11},
+               {0x09e3, 0x0a01, 30},
+               {0x0a02, 0x0a03, 1},
+               {0x0a3c, 0x0a3e, 2},
+               {0x0a3f, 0x0a42, 1},
+               {0x0a47, 0x0a48, 1},
+               {0x0a4b, 0x0a4d, 1},
+               {0x0a51, 0x0a70, 31},
+               {0x0a71, 0x0a75, 4},
+               {0x0a81, 0x0a83, 1},
+               {0x0abc, 0x0abe, 2},
+               {0x0abf, 0x0ac5, 1},
+               {0x0ac7, 0x0ac9, 1},
+               {0x0acb, 0x0acd, 1},
+               {0x0ae2, 0x0ae3, 1},
+               {0x0b01, 0x0b03, 1},
+               {0x0b3c, 0x0b3e, 2},
+               {0x0b3f, 0x0b44, 1},
+               {0x0b47, 0x0b48, 1},
+               {0x0b4b, 0x0b4d, 1},
+               {0x0b56, 0x0b57, 1},
+               {0x0b62, 0x0b63, 1},
+               {0x0b82, 0x0bbe, 60},
+               {0x0bbf, 0x0bc2, 1},
+               {0x0bc6, 0x0bc8, 1},
+               {0x0bca, 0x0bcd, 1},
+               {0x0bd7, 0x0c01, 42},
+               {0x0c02, 0x0c03, 1},
+               {0x0c3e, 0x0c44, 1},
+               {0x0c46, 0x0c48, 1},
+               {0x0c4a, 0x0c4d, 1},
+               {0x0c55, 0x0c56, 1},
+               {0x0c62, 0x0c63, 1},
+               {0x0c82, 0x0c83, 1},
+               {0x0cbc, 0x0cbe, 2},
+               {0x0cbf, 0x0cc4, 1},
+               {0x0cc6, 0x0cc8, 1},
+               {0x0cca, 0x0ccd, 1},
+               {0x0cd5, 0x0cd6, 1},
+               {0x0ce2, 0x0ce3, 1},
+               {0x0d02, 0x0d03, 1},
+               {0x0d3e, 0x0d44, 1},
+               {0x0d46, 0x0d48, 1},
+               {0x0d4a, 0x0d4d, 1},
+               {0x0d57, 0x0d62, 11},
+               {0x0d63, 0x0d82, 31},
+               {0x0d83, 0x0dca, 71},
+               {0x0dcf, 0x0dd4, 1},
+               {0x0dd6, 0x0dd8, 2},
+               {0x0dd9, 0x0ddf, 1},
+               {0x0df2, 0x0df3, 1},
+               {0x0e31, 0x0e34, 3},
+               {0x0e35, 0x0e3a, 1},
+               {0x0e47, 0x0e4e, 1},
+               {0x0eb1, 0x0eb4, 3},
+               {0x0eb5, 0x0eb9, 1},
+               {0x0ebb, 0x0ebc, 1},
+               {0x0ec8, 0x0ecd, 1},
+               {0x0f18, 0x0f19, 1},
+               {0x0f35, 0x0f39, 2},
+               {0x0f3e, 0x0f3f, 1},
+               {0x0f71, 0x0f84, 1},
+               {0x0f86, 0x0f87, 1},
+               {0x0f8d, 0x0f97, 1},
+               {0x0f99, 0x0fbc, 1},
+               {0x0fc6, 0x102b, 101},
+               {0x102c, 0x103e, 1},
+               {0x1056, 0x1059, 1},
+               {0x105e, 0x1060, 1},
+               {0x1062, 0x1064, 1},
+               {0x1067, 0x106d, 1},
+               {0x1071, 0x1074, 1},
+               {0x1082, 0x108d, 1},
+               {0x108f, 0x109a, 11},
+               {0x109b, 0x109d, 1},
+               {0x135d, 0x135f, 1},
+               {0x1712, 0x1714, 1},
+               {0x1732, 0x1734, 1},
+               {0x1752, 0x1753, 1},
+               {0x1772, 0x1773, 1},
+               {0x17b6, 0x17d3, 1},
+               {0x17dd, 0x180b, 46},
+               {0x180c, 0x180d, 1},
+               {0x18a9, 0x1920, 119},
+               {0x1921, 0x192b, 1},
+               {0x1930, 0x193b, 1},
+               {0x19b0, 0x19c0, 1},
+               {0x19c8, 0x19c9, 1},
+               {0x1a17, 0x1a1b, 1},
+               {0x1a55, 0x1a5e, 1},
+               {0x1a60, 0x1a7c, 1},
+               {0x1a7f, 0x1b00, 129},
+               {0x1b01, 0x1b04, 1},
+               {0x1b34, 0x1b44, 1},
+               {0x1b6b, 0x1b73, 1},
+               {0x1b80, 0x1b82, 1},
+               {0x1ba1, 0x1baa, 1},
+               {0x1be6, 0x1bf3, 1},
+               {0x1c24, 0x1c37, 1},
+               {0x1cd0, 0x1cd2, 1},
+               {0x1cd4, 0x1ce8, 1},
+               {0x1ced, 0x1cf2, 5},
+               {0x1dc0, 0x1de6, 1},
+               {0x1dfc, 0x1dff, 1},
+               {0x20d0, 0x20f0, 1},
+               {0x2cef, 0x2cf1, 1},
+               {0x2d7f, 0x2de0, 97},
+               {0x2de1, 0x2dff, 1},
+               {0x302a, 0x302f, 1},
+               {0x3099, 0x309a, 1},
+               {0xa66f, 0xa672, 1},
+               {0xa67c, 0xa67d, 1},
+               {0xa6f0, 0xa6f1, 1},
+               {0xa802, 0xa806, 4},
+               {0xa80b, 0xa823, 24},
+               {0xa824, 0xa827, 1},
+               {0xa880, 0xa881, 1},
+               {0xa8b4, 0xa8c4, 1},
+               {0xa8e0, 0xa8f1, 1},
+               {0xa926, 0xa92d, 1},
+               {0xa947, 0xa953, 1},
+               {0xa980, 0xa983, 1},
+               {0xa9b3, 0xa9c0, 1},
+               {0xaa29, 0xaa36, 1},
+               {0xaa43, 0xaa4c, 9},
+               {0xaa4d, 0xaa7b, 46},
+               {0xaab0, 0xaab2, 2},
+               {0xaab3, 0xaab4, 1},
+               {0xaab7, 0xaab8, 1},
+               {0xaabe, 0xaabf, 1},
+               {0xaac1, 0xabe3, 290},
+               {0xabe4, 0xabea, 1},
+               {0xabec, 0xabed, 1},
+               {0xfb1e, 0xfe00, 738},
+               {0xfe01, 0xfe0f, 1},
+               {0xfe20, 0xfe26, 1},
+       },
+       R32: []Range32{
+               {0x101fd, 0x10a01, 2052},
+               {0x10a02, 0x10a03, 1},
+               {0x10a05, 0x10a06, 1},
+               {0x10a0c, 0x10a0f, 1},
+               {0x10a38, 0x10a3a, 1},
+               {0x10a3f, 0x11000, 1473},
+               {0x11001, 0x11002, 1},
+               {0x11038, 0x11046, 1},
+               {0x11080, 0x11082, 1},
+               {0x110b0, 0x110ba, 1},
+               {0x1d165, 0x1d169, 1},
+               {0x1d16d, 0x1d172, 1},
+               {0x1d17b, 0x1d182, 1},
+               {0x1d185, 0x1d18b, 1},
+               {0x1d1aa, 0x1d1ad, 1},
+               {0x1d242, 0x1d244, 1},
+               {0xe0100, 0xe01ef, 1},
+       },
+}
+
+var _Mc = &RangeTable{
+       R16: []Range16{
+               {0x0903, 0x093b, 56},
+               {0x093e, 0x0940, 1},
+               {0x0949, 0x094c, 1},
+               {0x094e, 0x094f, 1},
+               {0x0982, 0x0983, 1},
+               {0x09be, 0x09c0, 1},
+               {0x09c7, 0x09c8, 1},
+               {0x09cb, 0x09cc, 1},
+               {0x09d7, 0x0a03, 44},
+               {0x0a3e, 0x0a40, 1},
+               {0x0a83, 0x0abe, 59},
+               {0x0abf, 0x0ac0, 1},
+               {0x0ac9, 0x0acb, 2},
+               {0x0acc, 0x0b02, 54},
+               {0x0b03, 0x0b3e, 59},
+               {0x0b40, 0x0b47, 7},
+               {0x0b48, 0x0b4b, 3},
+               {0x0b4c, 0x0b57, 11},
+               {0x0bbe, 0x0bbf, 1},
+               {0x0bc1, 0x0bc2, 1},
+               {0x0bc6, 0x0bc8, 1},
+               {0x0bca, 0x0bcc, 1},
+               {0x0bd7, 0x0c01, 42},
+               {0x0c02, 0x0c03, 1},
+               {0x0c41, 0x0c44, 1},
+               {0x0c82, 0x0c83, 1},
+               {0x0cbe, 0x0cc0, 2},
+               {0x0cc1, 0x0cc4, 1},
+               {0x0cc7, 0x0cc8, 1},
+               {0x0cca, 0x0ccb, 1},
+               {0x0cd5, 0x0cd6, 1},
+               {0x0d02, 0x0d03, 1},
+               {0x0d3e, 0x0d40, 1},
+               {0x0d46, 0x0d48, 1},
+               {0x0d4a, 0x0d4c, 1},
+               {0x0d57, 0x0d82, 43},
+               {0x0d83, 0x0dcf, 76},
+               {0x0dd0, 0x0dd1, 1},
+               {0x0dd8, 0x0ddf, 1},
+               {0x0df2, 0x0df3, 1},
+               {0x0f3e, 0x0f3f, 1},
+               {0x0f7f, 0x102b, 172},
+               {0x102c, 0x1031, 5},
+               {0x1038, 0x103b, 3},
+               {0x103c, 0x1056, 26},
+               {0x1057, 0x1062, 11},
+               {0x1063, 0x1064, 1},
+               {0x1067, 0x106d, 1},
+               {0x1083, 0x1084, 1},
+               {0x1087, 0x108c, 1},
+               {0x108f, 0x109a, 11},
+               {0x109b, 0x109c, 1},
+               {0x17b6, 0x17be, 8},
+               {0x17bf, 0x17c5, 1},
+               {0x17c7, 0x17c8, 1},
+               {0x1923, 0x1926, 1},
+               {0x1929, 0x192b, 1},
+               {0x1930, 0x1931, 1},
+               {0x1933, 0x1938, 1},
+               {0x19b0, 0x19c0, 1},
+               {0x19c8, 0x19c9, 1},
+               {0x1a19, 0x1a1b, 1},
+               {0x1a55, 0x1a57, 2},
+               {0x1a61, 0x1a63, 2},
+               {0x1a64, 0x1a6d, 9},
+               {0x1a6e, 0x1a72, 1},
+               {0x1b04, 0x1b35, 49},
+               {0x1b3b, 0x1b3d, 2},
+               {0x1b3e, 0x1b41, 1},
+               {0x1b43, 0x1b44, 1},
+               {0x1b82, 0x1ba1, 31},
+               {0x1ba6, 0x1ba7, 1},
+               {0x1baa, 0x1be7, 61},
+               {0x1bea, 0x1bec, 1},
+               {0x1bee, 0x1bf2, 4},
+               {0x1bf3, 0x1c24, 49},
+               {0x1c25, 0x1c2b, 1},
+               {0x1c34, 0x1c35, 1},
+               {0x1ce1, 0x1cf2, 17},
+               {0xa823, 0xa824, 1},
+               {0xa827, 0xa880, 89},
+               {0xa881, 0xa8b4, 51},
+               {0xa8b5, 0xa8c3, 1},
+               {0xa952, 0xa953, 1},
+               {0xa983, 0xa9b4, 49},
+               {0xa9b5, 0xa9ba, 5},
+               {0xa9bb, 0xa9bd, 2},
+               {0xa9be, 0xa9c0, 1},
+               {0xaa2f, 0xaa30, 1},
+               {0xaa33, 0xaa34, 1},
+               {0xaa4d, 0xaa7b, 46},
+               {0xabe3, 0xabe4, 1},
+               {0xabe6, 0xabe7, 1},
+               {0xabe9, 0xabea, 1},
+               {0xabec, 0xabec, 1},
+       },
+       R32: []Range32{
+               {0x11000, 0x11000, 1},
+               {0x11002, 0x11082, 128},
+               {0x110b0, 0x110b2, 1},
+               {0x110b7, 0x110b8, 1},
+               {0x1d165, 0x1d166, 1},
+               {0x1d16d, 0x1d172, 1},
+       },
+}
+
+var _Me = &RangeTable{
+       R16: []Range16{
+               {0x0488, 0x0489, 1},
+               {0x20dd, 0x20e0, 1},
+               {0x20e2, 0x20e4, 1},
+               {0xa670, 0xa672, 1},
+       },
+}
+
+var _Mn = &RangeTable{
+       R16: []Range16{
+               {0x0300, 0x036f, 1},
+               {0x0483, 0x0487, 1},
+               {0x0591, 0x05bd, 1},
+               {0x05bf, 0x05c1, 2},
+               {0x05c2, 0x05c4, 2},
+               {0x05c5, 0x05c7, 2},
+               {0x0610, 0x061a, 1},
+               {0x064b, 0x065f, 1},
+               {0x0670, 0x06d6, 102},
+               {0x06d7, 0x06dc, 1},
+               {0x06df, 0x06e4, 1},
+               {0x06e7, 0x06e8, 1},
+               {0x06ea, 0x06ed, 1},
+               {0x0711, 0x0730, 31},
+               {0x0731, 0x074a, 1},
+               {0x07a6, 0x07b0, 1},
+               {0x07eb, 0x07f3, 1},
+               {0x0816, 0x0819, 1},
+               {0x081b, 0x0823, 1},
+               {0x0825, 0x0827, 1},
+               {0x0829, 0x082d, 1},
+               {0x0859, 0x085b, 1},
+               {0x0900, 0x0902, 1},
+               {0x093a, 0x093c, 2},
+               {0x0941, 0x0948, 1},
+               {0x094d, 0x0951, 4},
+               {0x0952, 0x0957, 1},
+               {0x0962, 0x0963, 1},
+               {0x0981, 0x09bc, 59},
+               {0x09c1, 0x09c4, 1},
+               {0x09cd, 0x09e2, 21},
+               {0x09e3, 0x0a01, 30},
+               {0x0a02, 0x0a3c, 58},
+               {0x0a41, 0x0a42, 1},
+               {0x0a47, 0x0a48, 1},
+               {0x0a4b, 0x0a4d, 1},
+               {0x0a51, 0x0a70, 31},
+               {0x0a71, 0x0a75, 4},
+               {0x0a81, 0x0a82, 1},
+               {0x0abc, 0x0ac1, 5},
+               {0x0ac2, 0x0ac5, 1},
+               {0x0ac7, 0x0ac8, 1},
+               {0x0acd, 0x0ae2, 21},
+               {0x0ae3, 0x0b01, 30},
+               {0x0b3c, 0x0b3f, 3},
+               {0x0b41, 0x0b44, 1},
+               {0x0b4d, 0x0b56, 9},
+               {0x0b62, 0x0b63, 1},
+               {0x0b82, 0x0bc0, 62},
+               {0x0bcd, 0x0c3e, 113},
+               {0x0c3f, 0x0c40, 1},
+               {0x0c46, 0x0c48, 1},
+               {0x0c4a, 0x0c4d, 1},
+               {0x0c55, 0x0c56, 1},
+               {0x0c62, 0x0c63, 1},
+               {0x0cbc, 0x0cbf, 3},
+               {0x0cc6, 0x0ccc, 6},
+               {0x0ccd, 0x0ce2, 21},
+               {0x0ce3, 0x0d41, 94},
+               {0x0d42, 0x0d44, 1},
+               {0x0d4d, 0x0d62, 21},
+               {0x0d63, 0x0dca, 103},
+               {0x0dd2, 0x0dd4, 1},
+               {0x0dd6, 0x0e31, 91},
+               {0x0e34, 0x0e3a, 1},
+               {0x0e47, 0x0e4e, 1},
+               {0x0eb1, 0x0eb4, 3},
+               {0x0eb5, 0x0eb9, 1},
+               {0x0ebb, 0x0ebc, 1},
+               {0x0ec8, 0x0ecd, 1},
+               {0x0f18, 0x0f19, 1},
+               {0x0f35, 0x0f39, 2},
+               {0x0f71, 0x0f7e, 1},
+               {0x0f80, 0x0f84, 1},
+               {0x0f86, 0x0f87, 1},
+               {0x0f8d, 0x0f97, 1},
+               {0x0f99, 0x0fbc, 1},
+               {0x0fc6, 0x102d, 103},
+               {0x102e, 0x1030, 1},
+               {0x1032, 0x1037, 1},
+               {0x1039, 0x103a, 1},
+               {0x103d, 0x103e, 1},
+               {0x1058, 0x1059, 1},
+               {0x105e, 0x1060, 1},
+               {0x1071, 0x1074, 1},
+               {0x1082, 0x1085, 3},
+               {0x1086, 0x108d, 7},
+               {0x109d, 0x135d, 704},
+               {0x135e, 0x135f, 1},
+               {0x1712, 0x1714, 1},
+               {0x1732, 0x1734, 1},
+               {0x1752, 0x1753, 1},
+               {0x1772, 0x1773, 1},
+               {0x17b7, 0x17bd, 1},
+               {0x17c6, 0x17c9, 3},
+               {0x17ca, 0x17d3, 1},
+               {0x17dd, 0x180b, 46},
+               {0x180c, 0x180d, 1},
+               {0x18a9, 0x1920, 119},
+               {0x1921, 0x1922, 1},
+               {0x1927, 0x1928, 1},
+               {0x1932, 0x1939, 7},
+               {0x193a, 0x193b, 1},
+               {0x1a17, 0x1a18, 1},
+               {0x1a56, 0x1a58, 2},
+               {0x1a59, 0x1a5e, 1},
+               {0x1a60, 0x1a62, 2},
+               {0x1a65, 0x1a6c, 1},
+               {0x1a73, 0x1a7c, 1},
+               {0x1a7f, 0x1b00, 129},
+               {0x1b01, 0x1b03, 1},
+               {0x1b34, 0x1b36, 2},
+               {0x1b37, 0x1b3a, 1},
+               {0x1b3c, 0x1b42, 6},
+               {0x1b6b, 0x1b73, 1},
+               {0x1b80, 0x1b81, 1},
+               {0x1ba2, 0x1ba5, 1},
+               {0x1ba8, 0x1ba9, 1},
+               {0x1be6, 0x1be8, 2},
+               {0x1be9, 0x1bed, 4},
+               {0x1bef, 0x1bf1, 1},
+               {0x1c2c, 0x1c33, 1},
+               {0x1c36, 0x1c37, 1},
+               {0x1cd0, 0x1cd2, 1},
+               {0x1cd4, 0x1ce0, 1},
+               {0x1ce2, 0x1ce8, 1},
+               {0x1ced, 0x1dc0, 211},
+               {0x1dc1, 0x1de6, 1},
+               {0x1dfc, 0x1dff, 1},
+               {0x20d0, 0x20dc, 1},
+               {0x20e1, 0x20e5, 4},
+               {0x20e6, 0x20f0, 1},
+               {0x2cef, 0x2cf1, 1},
+               {0x2d7f, 0x2de0, 97},
+               {0x2de1, 0x2dff, 1},
+               {0x302a, 0x302f, 1},
+               {0x3099, 0x309a, 1},
+               {0xa66f, 0xa67c, 13},
+               {0xa67d, 0xa6f0, 115},
+               {0xa6f1, 0xa802, 273},
+               {0xa806, 0xa80b, 5},
+               {0xa825, 0xa826, 1},
+               {0xa8c4, 0xa8e0, 28},
+               {0xa8e1, 0xa8f1, 1},
+               {0xa926, 0xa92d, 1},
+               {0xa947, 0xa951, 1},
+               {0xa980, 0xa982, 1},
+               {0xa9b3, 0xa9b6, 3},
+               {0xa9b7, 0xa9b9, 1},
+               {0xa9bc, 0xaa29, 109},
+               {0xaa2a, 0xaa2e, 1},
+               {0xaa31, 0xaa32, 1},
+               {0xaa35, 0xaa36, 1},
+               {0xaa43, 0xaa4c, 9},
+               {0xaab0, 0xaab2, 2},
+               {0xaab3, 0xaab4, 1},
+               {0xaab7, 0xaab8, 1},
+               {0xaabe, 0xaabf, 1},
+               {0xaac1, 0xabe5, 292},
+               {0xabe8, 0xabed, 5},
+               {0xfb1e, 0xfe00, 738},
+               {0xfe01, 0xfe0f, 1},
+               {0xfe20, 0xfe26, 1},
+       },
+       R32: []Range32{
+               {0x101fd, 0x10a01, 2052},
+               {0x10a02, 0x10a03, 1},
+               {0x10a05, 0x10a06, 1},
+               {0x10a0c, 0x10a0f, 1},
+               {0x10a38, 0x10a3a, 1},
+               {0x10a3f, 0x11001, 1474},
+               {0x11038, 0x11046, 1},
+               {0x11080, 0x11081, 1},
+               {0x110b3, 0x110b6, 1},
+               {0x110b9, 0x110ba, 1},
+               {0x1d167, 0x1d169, 1},
+               {0x1d17b, 0x1d182, 1},
+               {0x1d185, 0x1d18b, 1},
+               {0x1d1aa, 0x1d1ad, 1},
+               {0x1d242, 0x1d244, 1},
+               {0xe0100, 0xe01ef, 1},
+       },
+}
+
+var _N = &RangeTable{
+       R16: []Range16{
+               {0x0030, 0x0039, 1},
+               {0x00b2, 0x00b3, 1},
+               {0x00b9, 0x00bc, 3},
+               {0x00bd, 0x00be, 1},
+               {0x0660, 0x0669, 1},
+               {0x06f0, 0x06f9, 1},
+               {0x07c0, 0x07c9, 1},
+               {0x0966, 0x096f, 1},
+               {0x09e6, 0x09ef, 1},
+               {0x09f4, 0x09f9, 1},
+               {0x0a66, 0x0a6f, 1},
+               {0x0ae6, 0x0aef, 1},
+               {0x0b66, 0x0b6f, 1},
+               {0x0b72, 0x0b77, 1},
+               {0x0be6, 0x0bf2, 1},
+               {0x0c66, 0x0c6f, 1},
+               {0x0c78, 0x0c7e, 1},
+               {0x0ce6, 0x0cef, 1},
+               {0x0d66, 0x0d75, 1},
+               {0x0e50, 0x0e59, 1},
+               {0x0ed0, 0x0ed9, 1},
+               {0x0f20, 0x0f33, 1},
+               {0x1040, 0x1049, 1},
+               {0x1090, 0x1099, 1},
+               {0x1369, 0x137c, 1},
+               {0x16ee, 0x16f0, 1},
+               {0x17e0, 0x17e9, 1},
+               {0x17f0, 0x17f9, 1},
+               {0x1810, 0x1819, 1},
+               {0x1946, 0x194f, 1},
+               {0x19d0, 0x19da, 1},
+               {0x1a80, 0x1a89, 1},
+               {0x1a90, 0x1a99, 1},
+               {0x1b50, 0x1b59, 1},
+               {0x1bb0, 0x1bb9, 1},
+               {0x1c40, 0x1c49, 1},
+               {0x1c50, 0x1c59, 1},
+               {0x2070, 0x2074, 4},
+               {0x2075, 0x2079, 1},
+               {0x2080, 0x2089, 1},
+               {0x2150, 0x2182, 1},
+               {0x2185, 0x2189, 1},
+               {0x2460, 0x249b, 1},
+               {0x24ea, 0x24ff, 1},
+               {0x2776, 0x2793, 1},
+               {0x2cfd, 0x3007, 778},
+               {0x3021, 0x3029, 1},
+               {0x3038, 0x303a, 1},
+               {0x3192, 0x3195, 1},
+               {0x3220, 0x3229, 1},
+               {0x3251, 0x325f, 1},
+               {0x3280, 0x3289, 1},
+               {0x32b1, 0x32bf, 1},
+               {0xa620, 0xa629, 1},
+               {0xa6e6, 0xa6ef, 1},
+               {0xa830, 0xa835, 1},
+               {0xa8d0, 0xa8d9, 1},
+               {0xa900, 0xa909, 1},
+               {0xa9d0, 0xa9d9, 1},
+               {0xaa50, 0xaa59, 1},
+               {0xabf0, 0xabf9, 1},
+               {0xff10, 0xff19, 1},
+       },
+       R32: []Range32{
+               {0x10107, 0x10133, 1},
+               {0x10140, 0x10178, 1},
+               {0x1018a, 0x10320, 406},
+               {0x10321, 0x10323, 1},
+               {0x10341, 0x1034a, 9},
+               {0x103d1, 0x103d5, 1},
+               {0x104a0, 0x104a9, 1},
+               {0x10858, 0x1085f, 1},
+               {0x10916, 0x1091b, 1},
+               {0x10a40, 0x10a47, 1},
+               {0x10a7d, 0x10a7e, 1},
+               {0x10b58, 0x10b5f, 1},
+               {0x10b78, 0x10b7f, 1},
+               {0x10e60, 0x10e7e, 1},
+               {0x11052, 0x1106f, 1},
+               {0x12400, 0x12462, 1},
+               {0x1d360, 0x1d371, 1},
+               {0x1d7ce, 0x1d7ff, 1},
+               {0x1f100, 0x1f10a, 1},
+       },
+}
+
+var _Nd = &RangeTable{
+       R16: []Range16{
+               {0x0030, 0x0039, 1},
+               {0x0660, 0x0669, 1},
+               {0x06f0, 0x06f9, 1},
+               {0x07c0, 0x07c9, 1},
+               {0x0966, 0x096f, 1},
+               {0x09e6, 0x09ef, 1},
+               {0x0a66, 0x0a6f, 1},
+               {0x0ae6, 0x0aef, 1},
+               {0x0b66, 0x0b6f, 1},</