Description
Kopia version:
0.17.0 build: 89c8eb4
kopia consumes huge memory when "repository connect" command is executed. It consumes memory in the order of the gigabytes.
For the kopia cache of 1.3G, kopia command consumed approximately about 1.5G memory
I analyzed the code to see why it takes this much memory and found that the kopia command consumes memory in fetching the index blobs from repo to the index cache.
In the function fetchIndexBlobs it retrieves the missing index blobs from the repo using the function fetchOne(GetEncryptedBlob), this function reads the blobs and writes to the in memory buffer. Once it fetches whole blob it then writes to the file in the kopia cache directory using function addIndexBlob. If the index blobs size increases the memory consumption increases proportionately.
In the containerize environment on a openshift cluster the memory required for the pod of this cache size is 2.5G
Is there a why to reduce the memory consumption of the kopia command during repo connect?
Below is the kopia cache dump
4.0K /tmp/kopia_cache/contents
4.0K /tmp/kopia_cache/metadata
325M /tmp/kopia_cache/index-blobs/xs
309M /tmp/kopia_cache/index-blobs/xn
633M /tmp/kopia_cache/index-blobs
0 /tmp/kopia_cache/own-writes
16K /tmp/kopia_cache/blob-list
633M /tmp/kopia_cache/indexes
1.3G /tmp/kopia_cache
func (c *committedContentIndex) fetchIndexBlobs(ctx context.Context, isPermissiveCacheLoading bool, indexBlobs []blob.ID) error {
ch, err := c.missingIndexBlobs(ctx, indexBlobs)
if err != nil {
return err
}
if len(ch) == 0 {
return nil
}
eg, ctx := errgroup.WithContext(ctx)
for range parallelFetches {
eg.Go(func() error {
var data gather.WriteBuffer
defer data.Close()
for indexBlobID := range ch {
data.Reset()
if err := c.fetchOne(ctx, indexBlobID, &data); err != nil {
if isPermissiveCacheLoading {
c.log.Errorf("skipping bad read of index blob %v", indexBlobID)
continue
}
return errors.Wrapf(err, "error loading index blob %v", indexBlobID)
}
if err := c.addIndexBlob(ctx, indexBlobID, data.Bytes(), false); err != nil {
return errors.Wrap(err, "unable to add to committed content cache")
}
}
return nil
})
}
if err := eg.Wait(); err != nil {
return errors.Wrap(err, "error downloading indexes")
}
c.log.Debugf("Index blobs downloaded.")
return nil
}