diff --git a/api/v1/package.go b/api/v1/package.go index eda4f46..c635fff 100644 --- a/api/v1/package.go +++ b/api/v1/package.go @@ -12,9 +12,11 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "time" + "github.com/peterhellberg/link" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -106,19 +108,23 @@ func PushPackage(ctx context.Context, repos, distro, version string, fpath strin } type PackageDetail struct { - Name string `json:"name"` - DistroVersion string `json:"distro_version"` - CreateTime time.Time `json:"created_at"` - Version string `json:"version"` - Type string `json:"type"` - Filename string `json:"filename"` - UploaderName string `json:"uploader_name"` - Indexed bool `json:"indexed"` - PackageURL string `json:"package_url"` - DownloadURL string `json:"download_url"` + Name string `json:"name"` + Arch string `json:"architecture"` + Release string `json:"release"` + DistroVersion string `json:"distro_version"` + CreateTime time.Time `json:"created_at"` + Version string `json:"version"` + Type string `json:"type"` + Filename string `json:"filename"` + UploaderName string `json:"uploader_name"` + Indexed bool `json:"indexed"` + PackageURL string `json:"package_url"` + DownloadURL string `json:"download_url"` + DownloadsCountURL string `json:"downloads_count_url"` + DownloadsDetailURL string `json:"downloads_detail_url"` } -func SearchPackage(ctx context.Context, repos, distro, query, filter string) ([]PackageDetail, error) { +func SearchPackage(ctx context.Context, repos, distro string, perPage int, query, filter string) ([]PackageDetail, error) { q := url.Values{} if distro != "" { q.Add("dist", distro) @@ -126,37 +132,61 @@ func SearchPackage(ctx context.Context, repos, distro, query, filter string) ([] if query != "" { q.Add("q", query) } + if perPage != 0 { + q.Add("per_page", strconv.Itoa(perPage)) + } + if filter != "" { q.Add("filter", filter) } url := fmt.Sprintf("https://packagecloud.io/api/v1/repos/%s/search?%s", repos, q.Encode()) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, fmt.Errorf("http request: %s", err) - } - req.Header.Set("Accept", "application/json") + var webLink map[string]*link.Link + var details []PackageDetail - token := packagecloudToken(ctx) - req.SetBasicAuth(token, "") + var next = &link.Link{} + for ; next != nil; next = webLink["next"] { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("http request: %s", err) + } + req.Header.Set("Accept", "application/json") + token := packagecloudToken(ctx) + req.SetBasicAuth(token, "") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http post: %s", err) + } + defer resp.Body.Close() - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("http post: %s", err) - } - defer resp.Body.Close() + total := resp.Header.Get("Total") + perPage := resp.Header.Get("Per-Page") + totalInt, _ := strconv.Atoi(total) + perPageInt, _ := strconv.Atoi(perPage) + + if total != "" && perPage != "" && totalInt > perPageInt { + webLink = link.ParseResponse(resp) + if n, ok := webLink["next"]; ok { + url = n.URI + } + + } else { + next = nil + } - switch resp.StatusCode { - case http.StatusOK: - var details []PackageDetail - if err := json.NewDecoder(resp.Body).Decode(&details); err != nil { - return nil, fmt.Errorf("json decode: %s", err) + switch resp.StatusCode { + case http.StatusOK: + var detail []PackageDetail + if err := json.NewDecoder(resp.Body).Decode(&detail); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + details = append(details, detail...) + default: + b, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("resp: %s, %q", resp.Status, b) } - return details, nil - default: - b, _ := ioutil.ReadAll(resp.Body) - return nil, fmt.Errorf("resp: %s, %q", resp.Status, b) } + return details, nil } func PromotePackage(ctx context.Context, dstRepos, srcRepo, distro, version string, fpath string) error { diff --git a/api/v1/stats.go b/api/v1/stats.go new file mode 100644 index 0000000..da6b8bb --- /dev/null +++ b/api/v1/stats.go @@ -0,0 +1,130 @@ +package packagecloud + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/peterhellberg/link" +) + +type CountValue struct { + Value int `json:"value"` +} + +type PackageDownloads struct { + DownloadedAt time.Time `json:"downloaded_at"` + IpAddress string `json:"ip_address"` + UserAgent string `json:"user_agent"` + Source string `json:"source"` + ReadToken string `json:"read_token"` +} + +func GetDownloadCount(ctx context.Context, pkg PackageDetail, startDate, endDate string) (*CountValue, error) { + q := url.Values{} + if startDate != "" { + q.Add("start_date", startDate) + } + if endDate != "" { + q.Add("end_date", endDate) + } + u := &url.URL{ + Scheme: "https", + Host: "packagecloud.io", + Path: pkg.DownloadsCountURL, + RawQuery: q.Encode(), + } + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, fmt.Errorf("error http get: %v", err) + } + req.Header.Set("Accept", "application/json") + + token := packagecloudToken(ctx) + req.SetBasicAuth(token, "") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error http request: %v", err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + var count CountValue + if err := json.NewDecoder(resp.Body).Decode(&count); err != nil { + return nil, fmt.Errorf("json decode: %v", err) + } + return &count, nil + default: + b, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("resp_status: %s, %q", resp.Status, b) + } +} + +func GetDownloadDetail(ctx context.Context, pkg PackageDetail, startDate, endDate string) ([]PackageDownloads, error) { + q := url.Values{} + if startDate != "" { + q.Add("start_date", startDate) + } + if endDate != "" { + q.Add("end_date", endDate) + } + u := &url.URL{ + Scheme: "https", + Host: "packagecloud.io", + Path: pkg.DownloadsDetailURL, + RawQuery: q.Encode(), + } + reqURL := u.String() + var webLink map[string]*link.Link + var details []PackageDownloads + + var next = &link.Link{} + for ; next != nil; next = webLink["next"] { + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("http request: %s", err) + } + req.Header.Set("Accept", "application/json") + token := packagecloudToken(ctx) + req.SetBasicAuth(token, "") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("http get: %s", err) + } + defer resp.Body.Close() + + total := resp.Header.Get("Total") + perPage := resp.Header.Get("Per-Page") + totalInt, _ := strconv.Atoi(total) + perPageInt, _ := strconv.Atoi(perPage) + + if total != "" && perPage != "" && totalInt > perPageInt { + webLink = link.ParseResponse(resp) + if n, ok := webLink["next"]; ok { + reqURL = n.URI + } + + } else { + next = nil + } + switch resp.StatusCode { + case http.StatusOK: + var detail []PackageDownloads + if err := json.NewDecoder(resp.Body).Decode(&detail); err != nil { + return nil, fmt.Errorf("json decode: %s", err) + } + details = append(details, detail...) + default: + b, _ := ioutil.ReadAll(resp.Body) + return nil, fmt.Errorf("resp: %s, %q", resp.Status, b) + } + + } + return details, nil +} diff --git a/command.go b/command.go index 2e92922..b1da3d4 100644 --- a/command.go +++ b/command.go @@ -110,7 +110,7 @@ var searchPackageCommand = &commandBase{ return subcommands.ExitUsageError } query := f.Arg(1) - details, err := packagecloud.SearchPackage(ctx, repos, distro, query, "") + details, err := packagecloud.SearchPackage(ctx, repos, distro, 0, query, "") if err != nil { log.Println(err) return subcommands.ExitFailure @@ -138,7 +138,7 @@ var pullPackageCommand = &commandBase{ return subcommands.ExitUsageError } query := f.Arg(1) - details, err := packagecloud.SearchPackage(ctx, repos, distro, query, "") + details, err := packagecloud.SearchPackage(ctx, repos, distro, 0, query, "") if err != nil { log.Println(err) return subcommands.ExitFailure diff --git a/go.mod b/go.mod index e88511b..8cb85ce 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.12 require ( github.com/google/subcommands v1.2.0 github.com/mattn/go-zglob v0.0.4 + github.com/peterhellberg/link v1.2.0 // indirect google.golang.org/grpc v1.53.0 ) - -replace github.com/atotto/packagecloud => ./ diff --git a/go.sum b/go.sum index a59c310..7ca444e 100644 --- a/go.sum +++ b/go.sum @@ -539,6 +539,8 @@ github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM= github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= +github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= +github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=