add memory limiter · Entire

add memory limiter

488a575·

Soph·3mo ago·4 files·+188 added/-8 removed

Sessions

16bfc6ff6845View transcript

[?
this should work right, or did the syntax change:Codex·GPT-5.4·1 step](/content/gh/entireio/git-sync/session/019d85eb-8e2e-7e83-8d90-640dce44d5f9#timeline-16bfc6ff6845/index.html)

Changes

4

158 unmodified lines

159
160
161
162
162
163
164
165

158 unmodified lines

if err == nil {
        t.Fatal("expected materialized limit failure")
    }
    if !strings.Contains(err.Error(), "materialized push requires") {
    if !strings.Contains(err.Error(), "materialized") || !strings.Contains(err.Error(), "limit") {
        t.Fatalf("expected materialized limit error, got %v", err)
    }
}

Minternal/syncer/integration_test.go+1/-1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

package syncer

import (
    "fmt"
    "io"

"github.com/go-git/go-git/v6/plumbing"
    "github.com/go-git/go-git/v6/plumbing/storer"
)

type materializedFetchLimitError struct {
    count int
    limit int
}

func (e materializedFetchLimitError) Error() string {
    return fmt.Sprintf(
        "materialized fetch exceeded object limit after %d objects (limit %d); narrow refs, use an empty target/bootstrap, or raise --materialized-max-objects",
        e.count,
        e.limit,
    )
}

type limitedStorer struct {
    base  storer.Storer
    limit int
    count int
}

func newLimitedStorer(base storer.Storer, limit int) *limitedStorer {
    return &limitedStorer{base: base, limit: limit}
}

func effectiveMaterializedMaxObjects(limit int) int {
    if limit > 0 {
        return limit
    }
    return DefaultMaterializedMaxObjects
}

func (s *limitedStorer) RawObjectWriter(typ plumbing.ObjectType, sz int64) (io.WriteCloser, error) {
    w, err := s.base.RawObjectWriter(typ, sz)
    if err != nil {
        return nil, err
    }
    return &countingWriteCloser{parent: s, inner: w}, nil
}

func (s *limitedStorer) NewEncodedObject() plumbing.EncodedObject {
    return s.base.NewEncodedObject()
}

func (s *limitedStorer) SetEncodedObject(obj plumbing.EncodedObject) (plumbing.Hash, error) {
    if err := s.bump(); err != nil {
        return plumbing.ZeroHash, err
    }
    return s.base.SetEncodedObject(obj)
}

func (s *limitedStorer) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) {
    return s.base.EncodedObject(t, h)
}

func (s *limitedStorer) IterEncodedObjects(t plumbing.ObjectType) (storer.EncodedObjectIter, error) {
    return s.base.IterEncodedObjects(t)
}

func (s *limitedStorer) HasEncodedObject(h plumbing.Hash) error {
    return s.base.HasEncodedObject(h)
}

func (s *limitedStorer) EncodedObjectSize(h plumbing.Hash) (int64, error) {
    return s.base.EncodedObjectSize(h)
}

func (s *limitedStorer) AddAlternate(remote string) error {
    return s.base.AddAlternate(remote)
}

func (s *limitedStorer) SetReference(ref *plumbing.Reference) error {
    return s.base.SetReference(ref)
}

func (s *limitedStorer) CheckAndSetReference(newRef, old *plumbing.Reference) error {
    return s.base.CheckAndSetReference(newRef, old)
}

func (s *limitedStorer) Reference(name plumbing.ReferenceName) (*plumbing.Reference, error) {
    return s.base.Reference(name)
}

func (s *limitedStorer) IterReferences() (storer.ReferenceIter, error) {
    return s.base.IterReferences()
}

func (s *limitedStorer) RemoveReference(name plumbing.ReferenceName) error {
    return s.base.RemoveReference(name)
}

func (s *limitedStorer) CountLooseRefs() (int, error) {
    return s.base.CountLooseRefs()
}

func (s *limitedStorer) PackRefs() error {
    return s.base.PackRefs()
}

func (s *limitedStorer) bump() error {
    s.count++
    if s.count > s.limit {
        return materializedFetchLimitError{count: s.count, limit: s.limit}
    }
    return nil
}

type countingWriteCloser struct {
    parent *limitedStorer
    inner  io.WriteCloser
}

func (w *countingWriteCloser) Write(p []byte) (int, error) {
    return w.inner.Write(p)
}

func (w *countingWriteCloser) Close() error {
    if err := w.parent.bump(); err != nil {
        _ = w.inner.Close()
        return err
    }
    return w.inner.Close()
}

Ainternal/syncer/limited_storer.go+131

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

package syncer

import (
    "errors"
    "testing"

"github.com/go-git/go-git/v6/plumbing"
    "github.com/go-git/go-git/v6/storage/memory"
)

func TestLimitedStorerSetEncodedObjectStopsAtLimit(t *testing.T) {
    base := memory.NewStorage()
    store := newLimitedStorer(base, 1)

if _, err := store.SetEncodedObject(testBlob("one")); err != nil {
        t.Fatalf("first object should fit within limit: %v", err)
    }
    err := func() error {
        _, err := store.SetEncodedObject(testBlob("two"))
        return err
    }()
    var limitErr materializedFetchLimitError
    if !errors.As(err, &limitErr) {
        t.Fatalf("expected materialized fetch limit error, got %v", err)
    }
    if limitErr.limit != 1 {
        t.Fatalf("unexpected limit error payload: %+v", limitErr)
    }
}

func TestEffectiveMaterializedMaxObjects(t *testing.T) {
    if got := effectiveMaterializedMaxObjects(123); got != 123 {
        t.Fatalf("effectiveMaterializedMaxObjects(123) = %d, want 123", got)
    }
    if got := effectiveMaterializedMaxObjects(0); got != DefaultMaterializedMaxObjects {
        t.Fatalf("effectiveMaterializedMaxObjects(0) = %d, want %d", got, DefaultMaterializedMaxObjects)
    }
}

func testBlob(content string) plumbing.EncodedObject {
    obj := &plumbing.MemoryObject{}
    obj.SetType(plumbing.BlobObject)
    obj.SetSize(int64(len(content)))
    if _, err := obj.Write([]byte(content)); err != nil {
        panic(err)
    }
    return obj
}
}

Ainternal/syncer/limited_storer_test.go+48

316 unmodified lines

317
318
319
320
321
322
323
324
325
320
321
322
323
324
325
326
327
328
107 unmodified lines

436
437
438
439
439
440
441
442
443

316 unmodified lines

}

type targetSession struct {
    conn      *gitproto.Conn
    adv       *packp.AdvRefs
    refMap    map[plumbing.ReferenceName]plumbing.Hash
    features  gitproto.TargetFeatures
    policy    planner.RelayTargetPolicy
    pusher    gitproto.Pusher
    conn     *gitproto.Conn
    adv      *packp.AdvRefs
    refMap   map[plumbing.ReferenceName]plumbing.Hash
    features gitproto.TargetFeatures
    policy   planner.RelayTargetPolicy
    pusher   gitproto.Pusher
}

// newSession performs the shared setup: protocol validation, mapping validation,
107 unmodified lines

return Result{}, fmt.Errorf("init in-memory repository: %w", err)
    }
    gpDesired := convert.DesiredRefs(desiredRefs)
    if err := sourceService.FetchToStore(ctx, repo.Storer, s.sourceConn, gpDesired, targetRefMap); err != nil {
    fetchStore := storer.Storer(newLimitedStorer(repo.Storer, effectiveMaterializedMaxObjects(cfg.MaterializedMaxObjects)))
    if err := sourceService.FetchToStore(ctx, fetchStore, s.sourceConn, gpDesired, targetRefMap); err != nil {
        if !errors.Is(err, git.NoErrAlreadyUpToDate) {
            return Result{}, err
        }
    }
}

Minternal/syncer/syncer.go+8/-7