From 7fed925c7a62774a1c5ffa3f5cc32313418bc2f8 Mon Sep 17 00:00:00 2001 From: Evsyukov Denis Date: Tue, 21 Jan 2025 10:44:43 +0300 Subject: [PATCH] refactoring: closure to func --- internal/client/client.go | 62 ++++++++ .../prettylog}/prettylog.go | 0 linkding/linkding.go | 34 ++--- main.go | 14 +- pkg/linkding/linkding.go | 55 +++++++ pkg/pocket/pocket.go | 128 +++++++++++++++++ pocket/pocket.go | 136 +++++++----------- 7 files changed, 310 insertions(+), 119 deletions(-) create mode 100644 internal/client/client.go rename {prettylog => internal/prettylog}/prettylog.go (100%) create mode 100644 pkg/linkding/linkding.go create mode 100644 pkg/pocket/pocket.go diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 0000000..9aa662d --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,62 @@ +package client + +import ( + "fmt" + "io" + "net/http" + "time" + + "github.com/cenkalti/backoff/v4" +) + +type Response struct { + StatusCode int + Body string + Header http.Header + Err error +} + +func Request(request *http.Request) (*Response, error) { + var response Response + + opt := func() error { + return operation(request, &response) + } + + if err := backoff.Retry(opt, backoff.NewExponentialBackOff()); err != nil { + return nil, err + } + + return &response, nil +} + +func operation(request *http.Request, response *Response) error { + client := &http.Client{ + Timeout: time.Second * 10, + } + r, err := client.Do(request) + if err != nil { + response.Err = err + return fmt.Errorf("failed to send request: %w", err) + } + defer r.Body.Close() + + if r.StatusCode == http.StatusUnauthorized { + err = fmt.Errorf("unauthorized") + response.Err = err + return backoff.Permanent(err) + } + + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + response.Err = err + return fmt.Errorf("failed to read response body: %w", err) + } + response.Body = string(bodyBytes) + + response.StatusCode = r.StatusCode + response.Header = r.Header + response.Err = nil + + return nil +} diff --git a/prettylog/prettylog.go b/internal/prettylog/prettylog.go similarity index 100% rename from prettylog/prettylog.go rename to internal/prettylog/prettylog.go diff --git a/linkding/linkding.go b/linkding/linkding.go index d3c365b..28e3fa6 100644 --- a/linkding/linkding.go +++ b/linkding/linkding.go @@ -1,32 +1,22 @@ package linkding import ( - "errors" "fmt" "io" "net/http" "net/url" "strings" - "time" - "github.com/cenkalti/backoff/v4" "github.com/tidwall/sjson" + + "github.com/juev/sync/internal/client" ) type Linkding struct { request http.Request - client *http.Client } -var ( - ErrLinkdingUnauthorized = errors.New("Linkding Unauthorized") -) - func New(apiURL, token string) (*Linkding, error) { - client := &http.Client{ - Timeout: time.Second * 10, - } - u, err := url.Parse(apiURL) if err != nil { return nil, fmt.Errorf("failed parse linkding apiUrl: %w", err) @@ -39,7 +29,6 @@ func New(apiURL, token string) (*Linkding, error) { return &Linkding{ request: *request, - client: client, }, nil } @@ -48,19 +37,14 @@ func (l *Linkding) Add(u string) error { request := l.request request.Body = io.NopCloser(strings.NewReader(body)) - operation := func() error { - response, err := l.client.Do(&request) - if err != nil { - return fmt.Errorf("failed to send request to linkding: %w", err) - } - defer response.Body.Close() - - if response.StatusCode == http.StatusUnauthorized { - return backoff.Permanent(ErrLinkdingUnauthorized) - } + response, err := client.Request(&request) + if err != nil { + return fmt.Errorf("failed to send request to linkding: %w", err) + } - return nil + if response.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("got response %d; X-Error=[%s]", response.StatusCode, response.Header.Get("X-Error")) } - return backoff.Retry(operation, backoff.NewExponentialBackOff()) + return nil } diff --git a/main.go b/main.go index 5b231df..2922e93 100644 --- a/main.go +++ b/main.go @@ -10,9 +10,9 @@ import ( "syscall" "time" - "github.com/juev/sync/linkding" - "github.com/juev/sync/pocket" - "github.com/juev/sync/prettylog" + "github.com/juev/sync/internal/prettylog" + "github.com/juev/sync/pkg/linkding" + "github.com/juev/sync/pkg/pocket" ) const ( @@ -94,14 +94,14 @@ func main() { runProcess := func(since int64) int64 { slog.Debug("Processing", "since", since) links, newSince, err := pocketClient.Retrive(since) - if len(links) == 0 { - slog.Debug("No new data from Pocket") - return newSince - } if err != nil { slog.Error("Failed to retrieve Pocket data", "error", err) return since } + if len(links) == 0 { + slog.Debug("No new data from Pocket") + return newSince + } for _, link := range links { slog.Info("Processing", "resolved_url", link) if err := linkdingClient.Add(link); err != nil { diff --git a/pkg/linkding/linkding.go b/pkg/linkding/linkding.go new file mode 100644 index 0000000..e3ddaa3 --- /dev/null +++ b/pkg/linkding/linkding.go @@ -0,0 +1,55 @@ +package linkding + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/juev/sync/internal/client" + "github.com/tidwall/sjson" +) + +type Linkding struct { + request http.Request + client *http.Client +} + +var ( + ErrLinkdingUnauthorized = errors.New("Linkding Unauthorized") +) + +func New(apiURL, token string) (*Linkding, error) { + u, err := url.Parse(apiURL) + if err != nil { + return nil, fmt.Errorf("failed parse linkding apiUrl: %w", err) + } + u = u.ResolveReference(&url.URL{Path: "/api/bookmarks/"}) + + request, _ := http.NewRequest(http.MethodPost, u.String(), nil) + request.Header.Add("Authorization", "Token "+token) + request.Header.Add("Content-Type", "application/json") + + return &Linkding{ + request: *request, + }, nil +} + +func (l *Linkding) Add(u string) error { + body, _ := sjson.Set("", "url", u) + request := l.request + request.Body = io.NopCloser(strings.NewReader(body)) + + response, err := client.Request(&request) + if err != nil { + return fmt.Errorf("failed to send request to linkding: %w", err) + } + + if response.StatusCode == http.StatusUnauthorized { + return fmt.Errorf("got response %d; X-Error=[%s]", response.StatusCode, response.Header.Get("X-Error")) + } + + return nil +} diff --git a/pkg/pocket/pocket.go b/pkg/pocket/pocket.go new file mode 100644 index 0000000..fa38286 --- /dev/null +++ b/pkg/pocket/pocket.go @@ -0,0 +1,128 @@ +package pocket + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/juev/sync/internal/client" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +type Pocket struct { + ConsumerKey string `json:"consumer_key"` + AccessToken string `json:"access_token"` + State string `json:"state"` + DetailType string `json:"detailType"` + Count int `json:"count"` + Offset int `json:"offset"` + Total int `json:"total"` + body string + client *http.Client +} + +const ( + endpoint = "https://getpocket.com/v3/get" + pocketCount = 30 + pocketTotal = 1 + pocketDefaultOffset = 0 + pocketState = "unread" + pocketDetailType = "simple" +) + +func New(consumerKey, accessToken string) (*Pocket, error) { + body, _ := json.Marshal(&Pocket{ + ConsumerKey: consumerKey, + AccessToken: accessToken, + State: pocketState, + DetailType: pocketDetailType, + Count: pocketCount, + Offset: pocketDefaultOffset, + Total: pocketTotal, + }) + + client := &http.Client{ + Timeout: time.Second * 10, + } + + return &Pocket{ + body: string(body), + client: client, + }, nil +} + +func (p *Pocket) Retrive(since int64) ([]string, int64, error) { + offset := pocketDefaultOffset + var ( + newSince int64 + result []string + err error + ) + + count := pocketCount + for count > 0 { + var links []string + links, newSince, err = p.operation(since, offset) + if err != nil { + return nil, 0, err + } + count = len(links) + if count > 0 { + result = append(result, links...) + } + offset += pocketCount + } + + return result, newSince, nil +} + +func (p *Pocket) operation(since int64, offset int) ([]string, int64, error) { + request, _ := http.NewRequest(http.MethodPost, endpoint, nil) + request.Header.Add("Content-Type", "application/json") + request.Header.Add("X-Accept", "application/json") + + body := p.body + body, _ = sjson.Set(body, "since", since) + body, _ = sjson.Set(body, "offset", offset) + request.Body = io.NopCloser(strings.NewReader(body)) + response, err := client.Request(request) + if err != nil { + return nil, 0, err + } + + if response.StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("got response %d; X-Error=[%s]", response.StatusCode, response.Header.Get("X-Error")) + } + + bodyString := response.Body + + if gjson.Get(bodyString, "error").String() != "" { + return nil, 0, fmt.Errorf("got response %d; X-Error=[%s]", response.StatusCode, response.Header.Get("X-Error")) + } + + // Update since + newSince := gjson.Get(bodyString, "since").Int() + + if gjson.Get(bodyString, "status").Int() == 2 { + return nil, newSince, nil + } + + list := gjson.Get(bodyString, "list").Map() + var result []string + for k := range list { + value := list[k].String() + u := gjson.Get(value, "resolved_url") + if u.String() == "" { + u = gjson.Get(value, "given_url") + } + if u.Exists() { + result = append(result, u.String()) + } + } + + return result, newSince, nil +} diff --git a/pocket/pocket.go b/pocket/pocket.go index 1c479eb..bd33597 100644 --- a/pocket/pocket.go +++ b/pocket/pocket.go @@ -7,9 +7,8 @@ import ( "io" "net/http" "strings" - "time" - "github.com/cenkalti/backoff/v4" + "github.com/juev/sync/internal/client" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -23,7 +22,6 @@ type Pocket struct { Offset int `json:"offset"` Total int `json:"total"` body string - client *http.Client } const ( @@ -50,95 +48,13 @@ func New(consumerKey, accessToken string) (*Pocket, error) { Total: pocketTotal, }) - client := &http.Client{ - Timeout: time.Second * 10, - } - return &Pocket{ - body: string(body), - client: client, + body: string(body), }, nil } func (p *Pocket) Retrive(since int64) ([]string, int64, error) { - request, _ := http.NewRequest(http.MethodPost, endpoint, nil) - request.Header.Add("Content-Type", "application/json") - request.Header.Add("X-Accept", "application/json") - var newSince int64 - operation := func(offset int) ([]string, error) { - body := p.body - body, _ = sjson.Set(body, "since", since) - body, _ = sjson.Set(body, "offset", offset) - request.Body = io.NopCloser(strings.NewReader(body)) - response, err := p.client.Do(request) - if err != nil { - return nil, err - } - defer response.Body.Close() - - if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("got response %d; X-Error=[%s]", response.StatusCode, response.Header.Get("X-Error")) - } - - bodyBytes, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - bodyString := string(bodyBytes) - if e := gjson.Get(bodyString, "error").String(); e != "" { - return nil, ErrSomethingWentWrong - } - - // Update since - newSince = gjson.Get(bodyString, "since").Int() - - if gjson.Get(bodyString, "status").Int() == 2 { - return nil, nil - } - - list := gjson.Get(bodyString, "list").Map() - var result []string - for k := range list { - value := list[k].String() - u := gjson.Get(value, "resolved_url") - if u.String() == "" { - u = gjson.Get(value, "given_url") - } - if u.Exists() { - result = append(result, u.String()) - } - } - - return result, nil - } - - retrive := func(offset int) ([]string, error) { - var ( - err error - links []string - ) - - ticker := backoff.NewTicker(backoff.NewExponentialBackOff()) - defer ticker.Stop() - for range ticker.C { - links, err = operation(offset) - if errors.Is(err, ErrSomethingWentWrong) { - break - } - if err != nil { - continue - } - - break - } - - if err != nil { - return nil, err - } - - return links, nil - } offset := pocketDefaultOffset var ( @@ -149,7 +65,7 @@ func (p *Pocket) Retrive(since int64) ([]string, int64, error) { count := pocketCount for count > 0 { var links []string - links, err = retrive(offset) + links, newSince, err = p.request(since, offset) if err != nil { return nil, newSince, err } @@ -162,3 +78,49 @@ func (p *Pocket) Retrive(since int64) ([]string, int64, error) { return result, newSince, nil } + +func (p *Pocket) request(since int64, offset int) ([]string, int64, error) { + request, _ := http.NewRequest(http.MethodPost, endpoint, nil) + request.Header.Add("Content-Type", "application/json") + request.Header.Add("X-Accept", "application/json") + + body := p.body + body, _ = sjson.Set(body, "since", since) + body, _ = sjson.Set(body, "offset", offset) + request.Body = io.NopCloser(strings.NewReader(body)) + response, err := client.Request(request) + if err != nil { + return nil, 0, err + } + + if response.StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("got response %d; X-Error=[%s]", response.StatusCode, response.Header.Get("X-Error")) + } + + bodyString := response.Body + if e := gjson.Get(bodyString, "error").String(); e != "" { + return nil, 0, ErrSomethingWentWrong + } + + // Update since + newSince := gjson.Get(bodyString, "since").Int() + + if gjson.Get(bodyString, "status").Int() == 2 { + return nil, newSince, nil + } + + list := gjson.Get(bodyString, "list").Map() + var result []string + for k := range list { + value := list[k].String() + u := gjson.Get(value, "resolved_url") + if u.String() == "" { + u = gjson.Get(value, "given_url") + } + if u.Exists() { + result = append(result, u.String()) + } + } + + return result, newSince, nil +}