import: share flat-directory discovery across importers · Entire
import: share flat-directory discovery across importers
99522d8→main·
computermode·3w ago·7 files·+86 added/-168 removed
Add discoverSessionFiles (discover.go), a single home for the discovery rules every flat-directory importer repeated: resolver-rejected entries skipped, session-ID filter, lookback-window cutoff by transcript modtime, and path sort. Route claude, cursor, pi, factory, gemini, and copilot through it; each Discover drops from a ~30-line loop to a ~10-line call with a small resolver (jsonlSessionResolver for the
codex keeps its own Discover — it walks a recursive date-sharded tree and filters by session_meta cwd, which the flat helper does not model.
Pure refactor: discovery behavior and results are unchanged (verified against the importtest fixtures and the existing per-agent Discover tests).
Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com
Sessions
01c03669fe4dView transcript
Changes
7
cmd/entire/cli/agentimport
Mclaude.go+1/-30
Mcopilot.go+8/-27
Mcursor.go+4/-26
Adiscover.go+70
Mfactory.go+1/-28
Mgemini.go+1/-28
Mpi.go+1/-29
2 unmodified lines
3
4
5
6
6
8
7
8
9
24 unmodified lines
34
35
36
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
37
38
39
40
2 unmodified lines
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
24 unmodified lines
}
dir = d
}
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil // no transcripts for this repo
}
return nil, fmt.Errorf("read claude session dir: %w", err)
}
cutoff := now.AddDate(0, 0, -LookbackDays)
var out []SessionFile
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".jsonl") {
continue
}
stem := strings.TrimSuffix(e.Name(), ".jsonl")
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, stem) {
continue
}
info, err := e.Info()
if err != nil {
continue
}
if info.ModTime().Before(cutoff) {
continue
}
out = append(out, SessionFile{Path: filepath.Join(dir, e.Name()), SessionID: stem})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
return discoverSessionFiles(dir, now, sessionFilter, jsonlSessionResolver(".jsonl", identitySessionID))
}
// SplitTurns produces one Turn per user-prompt line. Token usage for each turn
Mcmd/entire/cli/agentimport/claude.go+1/-30
6 unmodified lines
7
8
9
10
11
10
11
12
29 unmodified lines
42
43
44
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
40
41
42
43
44
45
46
4 unmodified lines
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
26 unmodified lines
}
dir = d
}
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read copilot session dir: %w", err)
}
cutoff := now.AddDate(0, 0, -LookbackDays)
var out []SessionFile
for _, e := range entries {
stem, path := cursorSessionFile(dir, e)
if path == "" {
continue
}
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, stem) {
continue
}
info, statErr := os.Stat(path)
if statErr != nil || info.ModTime().Before(cutoff) {
continue
}
out = append(out, SessionFile{Path: path, SessionID: stem})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
return discoverSessionFiles(dir, now, sessionFilter, func(dir string, e os.DirEntry) (string, string, bool) {
if !e.IsDir() {
continue
return "", "", false
}
sessionID := e.Name()
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, sessionID) {
continue
}
path := filepath.Join(dir, sessionID, "events.jsonl")
info, statErr := os.Stat(path)
if statErr != nil || info.ModTime().Before(cutoff) {
continue
}
path := filepath.Join(dir, e.Name(), "events.jsonl")
if !copilotSessionInRepo(path, repoRoot) {
continue
return "", "", false
}
out = append(out, SessionFile{Path: path, SessionID: sessionID})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
return e.Name(), path, true
})
}
// copilotSessionInRepo reports whether the session's session.start event places
Mcmd/entire/cli/agentimport/copilot.go+8/-27
4 unmodified lines
5
6
7
8
8
9
10
26 unmodified lines
37
38
39
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
40
41
42
43
44
45
46
4 unmodified lines
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
26 unmodified lines
}
dir = d
}
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read cursor session dir: %w", err)
}
cutoff := now.AddDate(0, 0, -LookbackDays)
var out []SessionFile
for _, e := range entries {
stem, path := cursorSessionFile(dir, e)
if path == "" {
continue
}
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, stem) {
continue
}
info, statErr := os.Stat(path)
if statErr != nil || info.ModTime().Before(cutoff) {
continue
}
out = append(out, SessionFile{Path: path, SessionID: stem})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
return discoverSessionFiles(dir, now, sessionFilter, func(dir string, e os.DirEntry) (string, string, bool) {
id, path := cursorSessionFile(dir, e)
return id, path, path != ""
})
}
// cursorSessionFile maps a directory entry to a (sessionID, transcript path),
Mcmd/entire/cli/agentimport/cursor.go+4/-26
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
package agentimport
import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
)
// sessionResolver maps a directory entry under dir to a discovered session's
// (sessionID, transcript path). ok=false skips the entry — it is not a
// transcript this importer should import (wrong extension/layout, or rejected
// by an importer-specific predicate such as a repo match).
type sessionResolver func(dir string, e os.DirEntry) (sessionID, path string, ok bool)
// discoverSessionFiles lists transcripts in dir using the discovery rules every
// flat-directory importer shares: skip entries the resolver rejects, apply the
// session-ID filter, drop transcripts older than the lookback window (by the
// transcript file's modtime), and sort by path. A missing dir yields no
// sessions (not an error).
//
// codex does not use this — its sessions live in a recursively-walked,
// session_meta-filtered tree rather than a flat directory.
func discoverSessionFiles(dir string, now time.Time, sessionFilter []string, resolve sessionResolver) ([]SessionFile, error) {
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read session dir: %w", err)
}
cutoff := now.AddDate(0, 0, -LookbackDays)
var out []SessionFile
for _, e := range entries {
sessionID, path, ok := resolve(dir, e)
if !ok {
continue
}
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, sessionID) {
continue
}
info, statErr := os.Stat(path)
if statErr != nil || info.ModTime().Before(cutoff) {
continue
}
out = append(out, SessionFile{Path: path, SessionID: sessionID})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
}
// identitySessionID uses the file stem verbatim as the session ID — the common
// case for agents that name transcripts <sessionID><ext>.
func identitySessionID(stem string) string { return stem }
// jsonlSessionResolver returns a sessionResolver for the common flat layout:
// one <stem><ext> file per session. deriveID maps the file stem to the session
// ID (identity for most agents; pi derives a UUID suffix). Directories and
// non-matching extensions are skipped.
func jsonlSessionResolver(ext string, deriveID func(stem string) string) sessionResolver {
return func(dir string, e os.DirEntry) (string, string, bool) {
if e.IsDir() || !strings.HasSuffix(e.Name(), ext) {
return "", "", false
}
stem := strings.TrimSuffix(e.Name(), ext)
return deriveID(stem), filepath.Join(dir, e.Name()), true
}
}
Acmd/entire/cli/agentimport/discover.go+70
2 unmodified lines
3
4
5
6
6
8
9
7
8
9
24 unmodified lines
34
35
36
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
37
38
39
40
2 unmodified lines
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/entireio/cli/cmd/entire/cli/agent"
)
24 unmodified lines
}
dir = d
}
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read factory session dir: %w", err)
}
cutoff := now.AddDate(0, 0, -LookbackDays)
var out []SessionFile
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".jsonl") {
continue
}
stem := strings.TrimSuffix(e.Name(), ".jsonl")
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, stem) {
continue
}
info, statErr := e.Info()
if statErr != nil || info.ModTime().Before(cutoff) {
continue
}
out = append(out, SessionFile{Path: filepath.Join(dir, e.Name()), SessionID: stem})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
return discoverSessionFiles(dir, now, sessionFilter, jsonlSessionResolver(".jsonl", identitySessionID))
}
// SplitTurns produces one Turn per user-prompt envelope, bounded by the next.
Mcmd/entire/cli/agentimport/factory.go+1/-28
2 unmodified lines
3
4
5
6
7
8
6
7
8
24 unmodified lines
33
34
35
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
36
37
38
39
2 unmodified lines
import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/entireio/cli/cmd/entire/cli/agent"
)
24 unmodified lines
}
dir = d
}
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read gemini session dir: %w", err)
}
cutoff := now.AddDate(0, 0, -LookbackDays)
var out []SessionFile
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {
continue
}
stem := strings.TrimSuffix(e.Name(), ".json")
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, stem) {
continue
}
info, statErr := e.Info()
if statErr != nil || info.ModTime().Before(cutoff) {
continue
}
out = append(out, SessionFile{Path: filepath.Join(dir, e.Name()), SessionID: stem})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
return discoverSessionFiles(dir, now, sessionFilter, jsonlSessionResolver(".json", identitySessionID))
}
// SplitTurns returns a single Turn covering the whole session. Offsets are
Mcmd/entire/cli/agentimport/gemini.go+1/-28
2 unmodified lines
3
4
5
6
7
8
6
7
8
25 unmodified lines
34
35
36
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
37
38
39
40
2 unmodified lines
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/entireio/cli/cmd/entire/cli/agent"
)
25 unmodified lines
}
dir = d
}
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read pi session dir: %w", err)
}
cutoff := now.AddDate(0, 0, -LookbackDays)
var out []SessionFile
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".jsonl") {
continue
}
stem := strings.TrimSuffix(e.Name(), ".jsonl")
sessionID := piSessionID(stem)
if len(sessionFilter) > 0 && !slices.Contains(sessionFilter, sessionID) {
continue
}
info, statErr := e.Info()
if statErr != nil || info.ModTime().Before(cutoff) {
continue
}
out = append(out, SessionFile{Path: filepath.Join(dir, e.Name()), SessionID: sessionID})
}
slices.SortFunc(out, func(a, b SessionFile) int { return strings.Compare(a.Path, b.Path) })
return out, nil
return discoverSessionFiles(dir, now, sessionFilter, jsonlSessionResolver(".jsonl", piSessionID))
}
// piSessionID extracts the <uuid> portion of a "<timestamp>_<uuid>" file stem.