diff --git a/pkg/meta/dump.go b/pkg/meta/dump.go index 554e14bc0565..97328e030b7e 100644 --- a/pkg/meta/dump.go +++ b/pkg/meta/dump.go @@ -132,9 +132,9 @@ func parseHex(c byte) (byte, error) { } } -func unescape(s string) string { +func unescape(s string) []byte { if !strings.ContainsRune(s, '%') { - return s + return []byte(s) } p := []byte(s) @@ -152,7 +152,7 @@ func unescape(s string) string { p[n] = c n++ } - return string(p[:n]) + return p[:n] } func (de *DumpedEntry) writeJSON(bw *bufio.Writer, depth int) error { diff --git a/pkg/meta/load_dump_test.go b/pkg/meta/load_dump_test.go index 07683a96bdf9..adc52b3a5fa9 100644 --- a/pkg/meta/load_dump_test.go +++ b/pkg/meta/load_dump_test.go @@ -33,13 +33,13 @@ const sampleFile = "metadata.sample" const subSampleFile = "metadata-sub.sample" func TestEscape(t *testing.T) { - var cs []byte = []byte("hello 世界") + var cs = []byte("hello 世界") for i := 0; i < 256; i++ { cs = append(cs, byte(i)) } s := string(cs) r := unescape(escape(s)) - if r != s { + if bytes.Compare(r, cs) != 0 { t.Fatalf("expected %v, but got %v", s, r) } } diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index e1aedd0862b0..a4e416ef1711 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -3055,7 +3055,7 @@ func (m *redisMeta) loadEntry(e *DumpedEntry, cs *DumpedCounters, refs map[strin if len(e.Entries) > 0 { dentries := make(map[string]interface{}) for _, c := range e.Entries { - dentries[unescape(c.Name)] = m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode) + dentries[string(unescape(c.Name))] = m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode) } p.HSet(ctx, m.entryKey(inode), dentries) } diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 9a680f5596c8..2f446fa2d596 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -54,7 +54,7 @@ type counter struct { type edge struct { Parent Ino `xorm:"unique(edge) notnull"` - Name string `xorm:"unique(edge) notnull"` + Name []byte `xorm:"unique(edge) varbinary(255) notnull"` Inode Ino `xorm:"notnull"` Type uint8 `xorm:"notnull"` } @@ -77,7 +77,7 @@ type node struct { type namedNode struct { node `xorm:"extends"` - Name string + Name []byte `xorm:"varbinary(255)"` } type chunk struct { @@ -92,7 +92,7 @@ type chunkRef struct { } type symlink struct { Inode Ino `xorm:"pk"` - Target string `xorm:"varchar(4096) notnull"` + Target []byte `xorm:"varbinary(4096) notnull"` } type xattr struct { @@ -208,25 +208,6 @@ func (m *dbMeta) doDeleteSlice(chunkid uint64, size uint32) error { }) } -func (m *dbMeta) updateCollate() { - if r, err := m.db.Query("show create table jfs_edge"); err != nil { - logger.Fatalf("show table jfs_edge: %s", err.Error()) - } else { - createTable := string(r[0]["Create Table"]) - // the default collate is case-insensitive - if !strings.Contains(createTable, "SET utf8mb4 COLLATE utf8mb4_bin") { - _, err := m.db.Exec("alter table jfs_edge modify name varchar (255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL") - if err != nil && strings.Contains(err.Error(), "Error 1071: Specified key was too long; max key length is 767 bytes") { - // MySQL 5.6 supports key length up to 767 bytes, so reduce the length of name to 190 chars - _, err = m.db.Exec("alter table jfs_edge modify name varchar (190) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL") - } - if err != nil { - logger.Fatalf("update collate: %s", err) - } - } - } -} - func (m *dbMeta) Init(format Format, force bool) error { if err := m.db.Sync2(new(setting), new(counter)); err != nil { logger.Fatalf("create table setting, counter: %s", err) @@ -246,9 +227,6 @@ func (m *dbMeta) Init(format Format, force bool) error { if err := m.db.Sync2(new(flock), new(plock)); err != nil { logger.Fatalf("create table flock, plock: %s", err) } - if m.db.DriverName() == "mysql" { - m.updateCollate() - } var s = setting{Name: "format"} ok, err := m.db.Get(&s) @@ -358,9 +336,6 @@ func (m *dbMeta) doNewSession(sinfo []byte) error { if err = m.db.Sync2(new(flock), new(plock)); err != nil { return fmt.Errorf("update table flock, plock: %s", err) } - if m.db.DriverName() == "mysql" { - m.updateCollate() - } for { if err = m.txn(func(s *xorm.Session) error { @@ -624,7 +599,7 @@ func (m *dbMeta) doLookup(ctx Context, parent Ino, name string, inode *Ino, attr if attr != nil { dbSession = dbSession.Join("INNER", &node{}, "jfs_edge.inode=jfs_node.inode") } - nn := namedNode{node: node{Parent: parent}, Name: name} + nn := namedNode{node: node{Parent: parent}, Name: []byte(name)} exist, err := dbSession.Select("*").Get(&nn) if err != nil { return errno(err) @@ -982,7 +957,7 @@ func (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode if pn.Type != TypeDirectory { return syscall.ENOTDIR } - var e = edge{Parent: parent, Name: name} + var e = edge{Parent: parent, Name: []byte(name)} ok, err = s.Get(&e) if err != nil { return err @@ -1030,7 +1005,7 @@ func (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode } } - if err = mustInsert(s, &edge{parent, name, ino, _type}, &n); err != nil { + if err = mustInsert(s, &edge{parent, []byte(name), ino, _type}, &n); err != nil { return err } if parent != TrashInode { @@ -1039,7 +1014,7 @@ func (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode } } if _type == TypeSymlink { - if err = mustInsert(s, &symlink{Inode: ino, Target: path}); err != nil { + if err = mustInsert(s, &symlink{Inode: ino, Target: []byte(path)}); err != nil { return err } } @@ -1072,7 +1047,7 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno { if pn.Type != TypeDirectory { return syscall.ENOTDIR } - var e = edge{Parent: parent, Name: name} + var e = edge{Parent: parent, Name: []byte(name)} ok, err = s.Get(&e) if err != nil { return err @@ -1080,7 +1055,7 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno { if !ok && m.conf.CaseInsensi { if ee := m.resolveCase(ctx, parent, name); ee != nil { ok = true - e.Name = string(ee.Name) + e.Name = ee.Name e.Inode = ee.Inode e.Type = ee.Attr.Typ } @@ -1134,7 +1109,7 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string) syscall.Errno { return err } if trash > 0 { - if err = mustInsert(s, &edge{trash, fmt.Sprintf("%d-%d-%s", parent, e.Inode, e.Name), e.Inode, e.Type}); err != nil { + if err = mustInsert(s, &edge{trash, []byte(fmt.Sprintf("%d-%d-%s", parent, e.Inode, string(e.Name))), e.Inode, e.Type}); err != nil { return err } } @@ -1200,7 +1175,7 @@ func (m *dbMeta) doRmdir(ctx Context, parent Ino, name string) syscall.Errno { if pn.Type != TypeDirectory { return syscall.ENOTDIR } - var e = edge{Parent: parent, Name: name} + var e = edge{Parent: parent, Name: []byte(name)} ok, err = s.Get(&e) if err != nil { return err @@ -1209,7 +1184,7 @@ func (m *dbMeta) doRmdir(ctx Context, parent Ino, name string) syscall.Errno { if ee := m.resolveCase(ctx, parent, name); ee != nil { ok = true e.Inode = ee.Inode - e.Name = string(ee.Name) + e.Name = ee.Name e.Type = ee.Attr.Typ } } @@ -1256,7 +1231,7 @@ func (m *dbMeta) doRmdir(ctx Context, parent Ino, name string) syscall.Errno { if _, err = s.Cols("ctime", "parent").Update(&n, &node{Inode: n.Inode}); err != nil { return err } - if err = mustInsert(s, &edge{trash, fmt.Sprintf("%d-%d-%s", parent, e.Inode, e.Name), e.Inode, e.Type}); err != nil { + if err = mustInsert(s, &edge{trash, []byte(fmt.Sprintf("%d-%d-%s", parent, e.Inode, string(e.Name))), e.Inode, e.Type}); err != nil { return err } } else { @@ -1289,7 +1264,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst var dn node var newSpace, newInode int64 err := m.txn(func(s *xorm.Session) error { - var se = edge{Parent: parentSrc, Name: nameSrc} + var se = edge{Parent: parentSrc, Name: []byte(nameSrc)} ok, err := s.Get(&se) if err != nil { return err @@ -1299,13 +1274,13 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst ok = true se.Inode = e.Inode se.Type = e.Attr.Typ - se.Name = string(e.Name) + se.Name = e.Name } } if !ok { return syscall.ENOENT } - if parentSrc == parentDst && se.Name == nameDst { + if parentSrc == parentDst && string(se.Name) == nameDst { if inode != nil { *inode = se.Inode } @@ -1342,7 +1317,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst return syscall.ENOENT } - var de = edge{Parent: parentDst, Name: nameDst} + var de = edge{Parent: parentDst, Name: []byte(nameDst)} ok, err = s.Get(&de) if err != nil { return err @@ -1352,7 +1327,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst ok = true de.Inode = e.Inode de.Type = e.Attr.Typ - de.Name = string(e.Name) + de.Name = e.Name } } now := time.Now().UnixNano() / 1e3 @@ -1453,7 +1428,7 @@ func (m *dbMeta) doRename(ctx Context, parentSrc Ino, nameSrc string, parentDst return err } name := fmt.Sprintf("%d-%d-%s", parentDst, dino, de.Name) - if err = mustInsert(s, &edge{trash, name, dino, de.Type}); err != nil { + if err = mustInsert(s, &edge{trash, []byte(name), dino, de.Type}); err != nil { return err } } else if de.Type != TypeDirectory && dn.Nlink > 0 { @@ -1536,7 +1511,7 @@ func (m *dbMeta) doLink(ctx Context, inode, parent Ino, name string, attr *Attr) if pn.Type != TypeDirectory { return syscall.ENOTDIR } - var e = edge{Parent: parent, Name: name} + var e = edge{Parent: parent, Name: []byte(name)} ok, err = s.Get(&e) if err != nil { return err @@ -1563,7 +1538,7 @@ func (m *dbMeta) doLink(ctx Context, inode, parent Ino, name string, attr *Attr) n.Nlink++ n.Ctime = now - if err = mustInsert(s, &edge{Parent: parent, Name: name, Inode: inode, Type: n.Type}); err != nil { + if err = mustInsert(s, &edge{Parent: parent, Name: []byte(name), Inode: inode, Type: n.Type}); err != nil { return err } if _, err := s.Cols("mtime", "ctime").Update(&pn, &node{Inode: parent}); err != nil { @@ -1589,13 +1564,13 @@ func (m *dbMeta) doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry return errno(err) } for _, n := range nodes { - if n.Name == "" { + if len(n.Name) == 0 { logger.Errorf("Corrupt entry with empty name: inode %d parent %d", n.Inode, inode) continue } entry := &Entry{ Inode: n.Inode, - Name: []byte(n.Name), + Name: n.Name, Attr: &Attr{}, } if plus != 0 { @@ -2333,7 +2308,7 @@ func (m *dbMeta) dumpEntry(inode Ino, typ uint8) (*DumpedEntry, error) { if !ok { logger.Warnf("no link target for inode %d", inode) } - e.Symlink = l.Target + e.Symlink = string(l.Target) } return nil }) @@ -2382,7 +2357,7 @@ func (m *dbMeta) dumpEntryFast(inode Ino, typ uint8) *DumpedEntry { logger.Warnf("no link target for inode %d", inode) l = &symlink{} } - e.Symlink = l.Target + e.Symlink = string(l.Target) } return e } @@ -2409,7 +2384,8 @@ func (m *dbMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth i if err := tree.writeJsonWithOutEntry(bw, depth); err != nil { return err } - sort.Slice(edges, func(i, j int) bool { return edges[i].Name < edges[j].Name }) + + sort.Slice(edges, func(i, j int) bool { return bytes.Compare(edges[i].Name, edges[j].Name) == -1 }) for idx, e := range edges { var entry *DumpedEntry @@ -2426,7 +2402,7 @@ func (m *dbMeta) dumpDir(inode Ino, tree *DumpedEntry, bw *bufio.Writer, depth i continue } - entry.Name = e.Name + entry.Name = string(e.Name) if e.Type == TypeDirectory { err = m.dumpDir(e.Inode, entry, bw, depth+2, showProgress) } else { @@ -2713,7 +2689,7 @@ func (m *dbMeta) loadEntry(e *DumpedEntry, cs *DumpedCounters, refs map[uint64]* if len(e.Xattrs) > 0 { xattrs := make([]*xattr, 0, len(e.Xattrs)) for _, x := range e.Xattrs { - xattrs = append(xattrs, &xattr{inode, x.Name, []byte(unescape(x.Value))}) + xattrs = append(xattrs, &xattr{inode, x.Name, unescape(x.Value)}) } beans = append(beans, xattrs) } diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index 535384552455..e722e7de13f9 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -2314,7 +2314,7 @@ func (m *kvMeta) loadEntry(e *DumpedEntry, cs *DumpedCounters, refs map[string]i } else if attr.Typ == TypeDirectory { attr.Length = 4 << 10 for _, c := range e.Entries { - tx.set(m.entryKey(inode, unescape(c.Name)), m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode)) + tx.set(m.entryKey(inode, string(unescape(c.Name))), m.packEntry(typeFromString(c.Attr.Type), c.Attr.Inode)) } } else if attr.Typ == TypeSymlink { symL := unescape(e.Symlink)