Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 27 additions & 27 deletions README.md

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions assemble.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"os"
)

// InvalidSeedAction represent the action that we will take if a seed
// InvalidSeedAction represents the action that we will take if a seed
// happens to be invalid. There are currently three options:
// - fail with an error
// - skip the invalid seed and try to continue
Expand Down Expand Up @@ -108,7 +108,7 @@ func AssembleFile(ctx context.Context, name string, idx Index, s Store, seeds []
ChunksTotal: len(idx.Chunks),
}

// Determine is the target exists and create it if not
// Determine if the target exists and create it if not
info, err := os.Stat(name)
switch {
case os.IsNotExist(err): // File doesn't exist yet => create it
Expand Down Expand Up @@ -147,7 +147,7 @@ func AssembleFile(ctx context.Context, name string, idx Index, s Store, seeds []
defer ns.close()
seeds = append([]Seed{ns}, seeds...)

// Start a self-seed which will become usable once chunks are written contigously
// Start a self-seed which will become usable once chunks are written contiguously
// beginning at position 0. There is no need to add this to the seeds list because
// when we create a plan it will be empty.
ss, err := newSelfSeed(name, idx)
Expand Down
2 changes: 1 addition & 1 deletion assemble_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func TestExtract(t *testing.T) {
}
os.Remove(out1.Name())

// This one is a complete file matching what we exepct at the end
// This one is a complete file matching what we expect at the end
out2, err := ioutil.TempFile("", "out2")
if err != nil {
t.Fatal(err)
Expand Down
2 changes: 1 addition & 1 deletion cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func (c Cache) Close() error {
return c.s.Close()
}

// New cache which GetChunk() function will return ChunkMissing error instead of ChunkInvalid
// RepairableCache is a cache whose GetChunk() function will return ChunkMissing error instead of ChunkInvalid
// so caller can redownload invalid chunk from store
type RepairableCache struct {
l WriteStore
Expand Down
2 changes: 1 addition & 1 deletion chunk.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func NewChunkWithID(id ChunkID, b []byte, skipVerify bool) (*Chunk, error) {
}

// NewChunkFromStorage builds a new chunk from data that is not in plain format.
// It uses raw storage format from it source and the modifiers are used to convert
// It uses raw storage format from its source and the modifiers are used to convert
// into plain data as needed.
func NewChunkFromStorage(id ChunkID, b []byte, modifiers Converters, skipVerify bool) (*Chunk, error) {
c := &Chunk{id: id, storage: b, converters: modifiers}
Expand Down
6 changes: 3 additions & 3 deletions chunker.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,12 +190,12 @@ func (c *Chunker) Next() (uint64, []byte, error) {

pos++

// didn't find a boundry before reaching the max?
// didn't find a boundary before reaching the max?
if pos >= m {
return c.split(pos, nil)
}

// Did we find a boundry?
// Did we find a boundary?
if c.hValue%c.hDiscriminator == c.hDiscriminator-1 {
return c.split(pos, nil)
}
Expand Down Expand Up @@ -247,7 +247,7 @@ func (c *Chunker) Avg() uint64 { return c.avg }
// Max returns the maximum chunk size
func (c *Chunker) Max() uint64 { return c.max }

// Hash implements the rolling hash algorithm used to find chunk bounaries
// Hash implements the rolling hash algorithm used to find chunk boundaries
// in a stream of bytes.
type Hash struct {
value uint32
Expand Down
2 changes: 1 addition & 1 deletion chunkstorage.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
)

// ChunkStorage stores chunks in a writable store. It can be safely used by multiple goroutines and
// contains an internal cache of what chunks have been store previously.
// contains an internal cache of what chunks have been stored previously.
type ChunkStorage struct {
sync.Mutex
ws WriteStore
Expand Down
4 changes: 2 additions & 2 deletions cmd/desync/chop.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ func newChopCommand(ctx context.Context) *cobra.Command {

cmd := &cobra.Command{
Use: "chop <index> <file>",
Short: "Reads chunks from a file according to an index",
Short: "Read chunks from a file according to an index",
Long: `Reads the index and extracts all referenced chunks from the file into a store,
local or remote.

Does not modify the input file or index in any. It's used to populate a chunk
Does not modify the input file or index in any way. It's used to populate a chunk
store by chopping up a file according to an existing index. To exclude chunks that
are known to exist in the target store already, use --ignore <index> which will
skip any chunks from the given index. The same can be achieved by providing the
Expand Down
12 changes: 6 additions & 6 deletions cmd/desync/chunkserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ func newChunkServerCommand(ctx context.Context) *cobra.Command {
reading from multiple local or remote stores as well as a local cache. If
--cert and --key are provided, the server will serve over HTTPS. The -w option
enables writing to this store, but this is only allowed when just one upstream
chunk store is provided. The option --skip-verify-write disables validation of
chunks written to this server which bypasses checksum validation as well as
the necessary decompression step to calculate it to improve performance. If -u
is used, only uncompressed chunks are being served (and accepted). If the
upstream store serves compressed chunks, everything will have to be decompressed
server-side so it's better to also read from uncompressed upstream stores.
chunk store is provided. The option --skip-verify-write disables hash validation
of chunks written to this server, avoiding the decompression step needed to
calculate checksums, to improve performance. If -u is used, only uncompressed
chunks are served (and accepted). If the upstream store serves compressed chunks,
everything will have to be decompressed server-side so it's better to also read
from uncompressed upstream stores.

While --concurrency does not limit the number of clients that can be served
concurrently, it does influence connection pools to remote upstream stores and
Expand Down
2 changes: 1 addition & 1 deletion cmd/desync/chunkserver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ func startChunkServer(t *testing.T, args ...string) (string, context.CancelFunc)
// Flush any handlers that were registered in the default mux before
http.DefaultServeMux = &http.ServeMux{}

// Start the server in a gorountine. Cancel the context when done
// Start the server in a goroutine. Cancel the context when done
ctx, cancel := context.WithCancel(context.Background())
cmd := newChunkServerCommand(ctx)
cmd.SetArgs(append(args, "-l", addr))
Expand Down
6 changes: 3 additions & 3 deletions cmd/desync/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func newConfigCommand(ctx context.Context) *cobra.Command {
Short: "Show or write config file",
Long: `Shows the current internal configuration settings, either the defaults,
the values from $HOME/.config/desync/config.json or the specified config file. The
output can be used to create a custom config file writing it to the specified file
output can be used to create a custom config file by writing it to the specified file
or $HOME/.config/desync/config.json by default.`,
Example: ` desync config
desync --config desync.json config -w`,
Expand Down Expand Up @@ -138,14 +138,14 @@ func runConfig(ctx context.Context, write bool) error {
return err
}

// Global config in the main packe defining the defaults. Those can be
// Global config in the main package defining the defaults. Those can be
// overridden by loading a config file or in the command line.
var cfg Config
var cfgFile string

// Look for $HOME/.config/desync and if present, load into the global config
// instance. Values defined in the file will be set accordingly, while anything
// that's not in the file will retain it's default values.
// that's not in the file will retain its default values.
func initConfig() {
var defaultLocation bool
if cfgFile == "" {
Expand Down
6 changes: 3 additions & 3 deletions cmd/desync/extract.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ set the path by writing the index file path, followed by a colon and the data pa
If several seed files and indexes are available, the -seed-dir option can be used
to automatically select all .caibx files in a directory as seeds. Use '-' to read
the index from STDIN. If a seed is invalid, by default the extract operation will be
aborted. With the -skip-invalid-seeds, the invalid seeds will be discarded and the
extraction will continue without them. Otherwise with the -regenerate-invalid-seeds,
the eventual invalid seed indexes will be regenerated, in memory, by using the
aborted. With --skip-invalid-seeds, the invalid seeds will be discarded and the
extraction will continue without them. Otherwise with --regenerate-invalid-seeds,
any invalid seed indexes will be regenerated, in memory, by using the
available data, and neither data nor indexes will be changed on disk. Also, if the seed changes
while processing, its invalid chunks will be taken from the self seed, or the store, instead
of aborting.`,
Expand Down
4 changes: 2 additions & 2 deletions cmd/desync/indexserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func newIndexServerCommand(ctx context.Context) *cobra.Command {
Use: "index-server",
Short: "Server for indexes over HTTP(S)",
Long: `Starts an HTTP index server that can be used as remote store. It supports
reading from a single local or a proxying to a remote store.
reading from a single local store or proxying to a remote store.
If --cert and --key are provided, the server will serve over HTTPS. The -w option
enables writing to this store.`,
Example: ` desync index-server -s sftp://192.168.1.1/indexes -l :8080`,
Expand Down Expand Up @@ -127,7 +127,7 @@ func serve(ctx context.Context, opt cmdServerOptions, addresses ...string) error
return err
}
if ok := certPool.AppendCertsFromPEM(b); !ok {
return errors.New("no client CA certficates found in client-ca file")
return errors.New("no client CA certificates found in client-ca file")
}
tlsConfig.ClientCAs = certPool
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/desync/indexserver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func startIndexServer(t *testing.T, args ...string) (string, context.CancelFunc)
// Flush any handlers that were registered in the default mux before
http.DefaultServeMux = &http.ServeMux{}

// Start the server in a gorountine. Cancel the context when done
// Start the server in a goroutine. Cancel the context when done
ctx, cancel := context.WithCancel(context.Background())
cmd := newIndexServerCommand(ctx)
cmd.SetArgs(append(args, "-l", addr))
Expand Down
4 changes: 2 additions & 2 deletions cmd/desync/mount-index.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ All chunks that are accessed by the mount are retrieved from the store and writt
the file as read operations are performed. Once all chunks have been accessed, the COR
file is fully populated. On termination, a <name>.state file is written containing
information about which chunks of the index have or have not been read. A state file is
only valid for a one cache-file and one index. When re-using it with a different index,
only valid for one cache file and one index. When re-using it with a different index,
data corruption can occur.

This command supports the --store-file option which can be used to define the stores
Expand All @@ -64,7 +64,7 @@ needing to restart the server. This can be done under load as well.
flags.StringVarP(&opt.corFile, "cor-file", "", "", "use a copy-on-read sparse file as cache")
flags.StringVarP(&opt.StateSaveFile, "cor-state-save", "", "", "file to store the state for copy-on-read")
flags.StringVarP(&opt.StateInitFile, "cor-state-init", "", "", "copy-on-read state init file")
flags.IntVarP(&opt.StateInitConcurrency, "cor-init-n", "", 10, "number of gorooutines to use for initialization (with --cor-state-init)")
flags.IntVarP(&opt.StateInitConcurrency, "cor-init-n", "", 10, "number of goroutines to use for initialization (with --cor-state-init)")
addStoreOptions(&opt.cmdStoreOptions, flags)
return cmd
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/desync/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func (o cmdServerOptions) validate() error {
func addServerOptions(o *cmdServerOptions, f *pflag.FlagSet) {
f.StringVar(&o.cert, "cert", "", "cert file in PEM format, requires --key")
f.StringVar(&o.key, "key", "", "key file in PEM format, requires --cert")
f.BoolVar(&o.mutualTLS, "mutual-tls", false, "require valid client certficate")
f.BoolVar(&o.mutualTLS, "mutual-tls", false, "require valid client certificate")
f.StringVar(&o.clientCA, "client-ca", "", "acceptable client certificate or CA")
f.StringVar(&o.auth, "authorization", "", "expected value of the authorization header in requests")
}
2 changes: 1 addition & 1 deletion cmd/desync/tar.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ less disk space is required as no intermediary catar is created. There
can however be a difference in performance depending on file size.

By default, input is read from local disk. Using --input-format=tar,
the input can be a tar file or stream to STDIN with '-'.
the input can be a tar file or a stream from STDIN with '-'.
`,
Example: ` desync tar documents.catar $HOME/Documents
desync tar -i -s /path/to/local pics.caidx $HOME/Pictures`,
Expand Down
2 changes: 1 addition & 1 deletion cmd/desync/verifyindex.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func newVerifyIndexCommand(ctx context.Context) *cobra.Command {

cmd := &cobra.Command{
Use: "verify-index <index> <file>",
Short: "Verifies an index matches a file",
Short: "Verify an index matches a file",
Long: `Verifies an index file matches the content of a blob. Use '-' to read the index
from STDIN.`,
Example: ` desync verify-index sftp://192.168.1.1/myIndex.caibx largefile.bin`,
Expand Down
2 changes: 1 addition & 1 deletion consoleindex.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (s ConsoleIndexStore) GetIndex(string) (i Index, e error) {
return IndexFromReader(os.Stdin)
}

// StoreIndex writes the provided indes to STDOUT. The name is ignored.
// StoreIndex writes the provided index to STDOUT. The name is ignored.
func (s ConsoleIndexStore) StoreIndex(name string, idx Index) error {
_, err := idx.WriteTo(os.Stdout)
return err
Expand Down
6 changes: 3 additions & 3 deletions coverter.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func (s Converters) fromStorage(in []byte) ([]byte, error) {
return b, nil
}

// Returns true is conversion involves compression. Typically
// Returns true if conversion involves compression. Typically
// used to determine the correct file-extension.
func (s Converters) hasCompression() bool {
for _, layer := range s {
Expand All @@ -65,12 +65,12 @@ func (s Converters) equal(c Converters) bool {

// converter is a storage data modifier layer.
type converter interface {
// Convert data from it's original form to storage format.
// Convert data from its original form to storage format.
// The input could be plain data, or the output of a prior
// converter.
toStorage([]byte) ([]byte, error)

// Convert data from it's storage format towards it's plain
// Convert data from its storage format towards its plain
// form. The input could be encrypted or compressed, while
// the output may be used for the next conversion layer.
fromStorage([]byte) ([]byte, error)
Expand Down
4 changes: 2 additions & 2 deletions dedupqueue.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ var _ Store = &DedupQueue{}
// DedupQueue wraps a store and provides deduplication of incoming chunk requests. This is useful when
// a burst of requests for the same chunk is received and the chunk store serving those is slow. With
// the DedupQueue wrapper, concurrent requests for the same chunk will result in just one request to the
// upstread store. Implements the Store interface.
// upstream store. Implements the Store interface.
type DedupQueue struct {
store Store
mu sync.Mutex
Expand Down Expand Up @@ -126,7 +126,7 @@ func (r *request) wait() (interface{}, error) {
return r.data, r.err
}

// Set the result data and marks this request as comlete.
// Set the result data and marks this request as complete.
func (r *request) markDone(data interface{}, err error) {
r.data = data
r.err = err
Expand Down
4 changes: 2 additions & 2 deletions digest.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@ type HashAlgorithm interface {
Algorithm() crypto.Hash
}

// SHA512-256 hashing algoritm for Digest.
// SHA512-256 hashing algorithm for Digest.
type SHA512256 struct{}

func (h SHA512256) Sum(data []byte) [32]byte { return sha512.Sum512_256(data) }
func (h SHA512256) Algorithm() crypto.Hash { return crypto.SHA512_256 }

// SHA256 hashing algoritm for Digest.
// SHA256 hashing algorithm for Digest.
type SHA256 struct{}

func (h SHA256) Sum(data []byte) [32]byte { return sha256.Sum256(data) }
Expand Down
2 changes: 1 addition & 1 deletion doc.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
Package desync implements data structures, protocols and features of
https://github.com/systemd/casync in order to allow support for additional
platforms and improve performace by way of concurrency and caching.
platforms and improve performance by way of concurrency and caching.

Supports the following casync data structures: catar archives, caibx/caidx index
files, castr stores (local or remote).
Expand Down
2 changes: 1 addition & 1 deletion failover.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func (g *FailoverGroup) current() (Store, int) {
return g.stores[g.active], g.active
}

// Fail over to the next available store after recveiving an error from i (the active). We
// Fail over to the next available store after receiving an error from i (the active). We
// need i to know which store returned the error as there could be failures from concurrent
// requests. Another request could have initiated the failover already. So ignore if i is not
// (no longer) the active store.
Expand Down
2 changes: 1 addition & 1 deletion fileseed.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func NewIndexSeed(dstFile string, srcFile string, index Index) (*FileSeed, error
// and a nil SeedSegment.
func (s *FileSeed) LongestMatchWith(chunks []IndexChunk) (int, SeedSegment) {
s.mu.RLock()
// isInvalid can be concurrently read or wrote. Use a mutex to avoid a race
// isInvalid can be concurrently read or written. Use a mutex to avoid a race
if len(chunks) == 0 || len(s.index.Chunks) == 0 || s.isInvalid {
return 0, nil
}
Expand Down
2 changes: 1 addition & 1 deletion gcs.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func (s GCStoreBase) String() string {
return s.Location
}

// Close the GCS base store. NOP opertation but needed to implement the store interface.
// Close the GCS base store. NOP operation but needed to implement the store interface.
func (s GCStoreBase) Close() error { return nil }

// NewGCStore creates a chunk store with Google Storage backing. The URL
Expand Down
4 changes: 2 additions & 2 deletions index.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func IndexFromReader(r io.Reader) (c Index, err error) {
c.Chunks[i].Start = lastOffset
c.Chunks[i].Size = r.Offset - lastOffset
lastOffset = r.Offset
// Check the max size of the chunk only. The min apperently doesn't apply
// Check the max size of the chunk only. The min apparently doesn't apply
// to the last chunk.
if c.Chunks[i].Size > c.Index.ChunkSizeMax {
return c, fmt.Errorf("chunk size %d is larger than maximum %d", c.Chunks[i].Size, c.Index.ChunkSizeMax)
Expand Down Expand Up @@ -180,7 +180,7 @@ func ChunkStream(ctx context.Context, c Chunker, ws WriteStore, n int) (Index, e

// Feed the workers, stop if there are any errors. To keep the index list in
// order, we calculate the checksum here before handing them over to the
// workers for compression and storage. That could probablybe optimized further
// workers for compression and storage. That could probably be optimized further
var num int // chunk #, so we can re-assemble the index in the right order later
loop:
for {
Expand Down
2 changes: 1 addition & 1 deletion ioctl_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const blkGetSize64 = 0x80081272
const fiCloneRange = 0x4020940d

// CanClone tries to determine if the filesystem allows cloning of blocks between
// two files. It'll create two tempfiles in the same dirs and attempt to perfom
// two files. It'll create two tempfiles in the same dirs and attempt to perform
// a 0-byte long block clone. If that's successful it'll return true.
func CanClone(dstFile, srcFile string) bool {
dst, err := ioutil.TempFile(filepath.Dir(dstFile), ".tmp")
Expand Down
4 changes: 2 additions & 2 deletions local.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func (s LocalStore) Verify(ctx context.Context, n int, repair bool, w io.Writer)
}()
}

// Go trough all chunks underneath Base, filtering out other files, then feed
// Go through all chunks underneath Base, filtering out other files, then feed
// the IDs to the workers
err := filepath.Walk(s.Base, func(path string, info os.FileInfo, err error) error {
// See if we're meant to stop
Expand Down Expand Up @@ -167,7 +167,7 @@ func (s LocalStore) Verify(ctx context.Context, n int, repair bool, w io.Writer)
// Prune removes any chunks from the store that are not contained in a list
// of chunks
func (s LocalStore) Prune(ctx context.Context, ids map[ChunkID]struct{}) error {
// Go trough all chunks underneath Base, filtering out other directories and files
// Go through all chunks underneath Base, filtering out other directories and files
err := filepath.Walk(s.Base, func(path string, info os.FileInfo, err error) error {
// See if we're meant to stop
select {
Expand Down
Loading