Add pagination support for search and add stats endpoint support

api/v1/package.go

@@ -12,9 +12,11 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/peterhellberg/link"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -106,57 +108,85 @@ func PushPackage(ctx context.Context, repos, distro, version string, fpath strin
 }
 
 type PackageDetail struct {
-	Name          string    `json:"name"`
-	DistroVersion string    `json:"distro_version"`
-	CreateTime    time.Time `json:"created_at"`
-	Version       string    `json:"version"`
-	Type          string    `json:"type"`
-	Filename      string    `json:"filename"`
-	UploaderName  string    `json:"uploader_name"`
-	Indexed       bool      `json:"indexed"`
-	PackageURL    string    `json:"package_url"`
-	DownloadURL   string    `json:"download_url"`
+	Name               string    `json:"name"`
+	Arch               string    `json:"architecture"`
+	Release            string    `json:"release"`
+	DistroVersion      string    `json:"distro_version"`
+	CreateTime         time.Time `json:"created_at"`
+	Version            string    `json:"version"`
+	Type               string    `json:"type"`
+	Filename           string    `json:"filename"`
+	UploaderName       string    `json:"uploader_name"`
+	Indexed            bool      `json:"indexed"`
+	PackageURL         string    `json:"package_url"`
+	DownloadURL        string    `json:"download_url"`
+	DownloadsCountURL  string    `json:"downloads_count_url"`
+	DownloadsDetailURL string    `json:"downloads_detail_url"`
 }
 
-func SearchPackage(ctx context.Context, repos, distro, query, filter string) ([]PackageDetail, error) {
+func SearchPackage(ctx context.Context, repos, distro string, perPage int, query, filter string) ([]PackageDetail, error) {
 	q := url.Values{}
 	if distro != "" {
 		q.Add("dist", distro)
 	}
 	if query != "" {
 		q.Add("q", query)
 	}
+	if perPage != 0 {
+		q.Add("per_page", strconv.Itoa(perPage))
+	}
+
 	if filter != "" {
 		q.Add("filter", filter)
 	}
 
 	url := fmt.Sprintf("https://packagecloud.io/api/v1/repos/%s/search?%s", repos, q.Encode())
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, fmt.Errorf("http request: %s", err)
-	}
-	req.Header.Set("Accept", "application/json")
+	var webLink map[string]*link.Link
+	var details []PackageDetail
 
-	token := packagecloudToken(ctx)
-	req.SetBasicAuth(token, "")
+	var next = &link.Link{}
+	for ; next != nil; next = webLink["next"] {
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			return nil, fmt.Errorf("http request: %s", err)
+		}
+		req.Header.Set("Accept", "application/json")
+		token := packagecloudToken(ctx)
+		req.SetBasicAuth(token, "")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("http post: %s", err)
+		}
+		defer resp.Body.Close()
 
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("http post: %s", err)
-	}
-	defer resp.Body.Close()
+		total := resp.Header.Get("Total")
+		perPage := resp.Header.Get("Per-Page")
+		totalInt, _ := strconv.Atoi(total)
+		perPageInt, _ := strconv.Atoi(perPage)
+
+		if total != "" && perPage != "" && totalInt > perPageInt {
+			webLink = link.ParseResponse(resp)
+			if n, ok := webLink["next"]; ok {
+				url = n.URI
+			}
+
+		} else {
+			next = nil
+		}
 
-	switch resp.StatusCode {
-	case http.StatusOK:
-		var details []PackageDetail
-		if err := json.NewDecoder(resp.Body).Decode(&details); err != nil {
-			return nil, fmt.Errorf("json decode: %s", err)
+		switch resp.StatusCode {
+		case http.StatusOK:
+			var detail []PackageDetail
+			if err := json.NewDecoder(resp.Body).Decode(&detail); err != nil {
+				return nil, fmt.Errorf("json decode: %s", err)
+			}
+			details = append(details, detail...)
+		default:
+			b, _ := ioutil.ReadAll(resp.Body)
+			return nil, fmt.Errorf("resp: %s, %q", resp.Status, b)
 		}
-		return details, nil
-	default:
-		b, _ := ioutil.ReadAll(resp.Body)
-		return nil, fmt.Errorf("resp: %s, %q", resp.Status, b)
 	}
+	return details, nil
 }
 
 func PromotePackage(ctx context.Context, dstRepos, srcRepo, distro, version string, fpath string) error {

api/v1/stats.go

@@ -0,0 +1,130 @@
+package packagecloud
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/peterhellberg/link"
+)
+
+type CountValue struct {
+	Value int `json:"value"`
+}
+
+type PackageDownloads struct {
+	DownloadedAt time.Time `json:"downloaded_at"`
+	IpAddress    string    `json:"ip_address"`
+	UserAgent    string    `json:"user_agent"`
+	Source       string    `json:"source"`
+	ReadToken    string    `json:"read_token"`
+}
+
+func GetDownloadCount(ctx context.Context, pkg PackageDetail, startDate, endDate string) (*CountValue, error) {
+	q := url.Values{}
+	if startDate != "" {
+		q.Add("start_date", startDate)
+	}
+	if endDate != "" {
+		q.Add("end_date", endDate)
+	}
+	u := &url.URL{
+		Scheme:   "https",
+		Host:     "packagecloud.io",
+		Path:     pkg.DownloadsCountURL,
+		RawQuery: q.Encode(),
+	}
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, fmt.Errorf("error http get: %v", err)
+	}
+	req.Header.Set("Accept", "application/json")
+
+	token := packagecloudToken(ctx)
+	req.SetBasicAuth(token, "")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error http request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		var count CountValue
+		if err := json.NewDecoder(resp.Body).Decode(&count); err != nil {
+			return nil, fmt.Errorf("json decode: %v", err)
+		}
+		return &count, nil
+	default:
+		b, _ := ioutil.ReadAll(resp.Body)
+		return nil, fmt.Errorf("resp_status: %s, %q", resp.Status, b)
+	}
+}
+
+func GetDownloadDetail(ctx context.Context, pkg PackageDetail, startDate, endDate string) ([]PackageDownloads, error) {
+	q := url.Values{}
+	if startDate != "" {
+		q.Add("start_date", startDate)
+	}
+	if endDate != "" {
+		q.Add("end_date", endDate)
+	}
+	u := &url.URL{
+		Scheme:   "https",
+		Host:     "packagecloud.io",
+		Path:     pkg.DownloadsDetailURL,
+		RawQuery: q.Encode(),
+	}
+	reqURL := u.String()
+	var webLink map[string]*link.Link
+	var details []PackageDownloads
+
+	var next = &link.Link{}
+	for ; next != nil; next = webLink["next"] {
+		req, err := http.NewRequest("GET", reqURL, nil)
+		if err != nil {
+			return nil, fmt.Errorf("http request: %s", err)
+		}
+		req.Header.Set("Accept", "application/json")
+		token := packagecloudToken(ctx)
+		req.SetBasicAuth(token, "")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("http get: %s", err)
+		}
+		defer resp.Body.Close()
+
+		total := resp.Header.Get("Total")
+		perPage := resp.Header.Get("Per-Page")
+		totalInt, _ := strconv.Atoi(total)
+		perPageInt, _ := strconv.Atoi(perPage)
+
+		if total != "" && perPage != "" && totalInt > perPageInt {
+			webLink = link.ParseResponse(resp)
+			if n, ok := webLink["next"]; ok {
+				reqURL = n.URI
+			}
+
+		} else {
+			next = nil
+		}
+		switch resp.StatusCode {
+		case http.StatusOK:
+			var detail []PackageDownloads
+			if err := json.NewDecoder(resp.Body).Decode(&detail); err != nil {
+				return nil, fmt.Errorf("json decode: %s", err)
+			}
+			details = append(details, detail...)
+		default:
+			b, _ := ioutil.ReadAll(resp.Body)
+			return nil, fmt.Errorf("resp: %s, %q", resp.Status, b)
+		}
+
+	}
+	return details, nil
+}

command.go

@@ -110,7 +110,7 @@ var searchPackageCommand = &commandBase{
 			return subcommands.ExitUsageError
 		}
 		query := f.Arg(1)
-		details, err := packagecloud.SearchPackage(ctx, repos, distro, query, "")
+		details, err := packagecloud.SearchPackage(ctx, repos, distro, 0, query, "")
 		if err != nil {
 			log.Println(err)
 			return subcommands.ExitFailure
@@ -138,7 +138,7 @@ var pullPackageCommand = &commandBase{
 			return subcommands.ExitUsageError
 		}
 		query := f.Arg(1)
-		details, err := packagecloud.SearchPackage(ctx, repos, distro, query, "")
+		details, err := packagecloud.SearchPackage(ctx, repos, distro, 0, query, "")
 		if err != nil {
 			log.Println(err)
 			return subcommands.ExitFailure

go.mod

@@ -5,7 +5,6 @@ go 1.12
 require (
 	github.com/google/subcommands v1.2.0
 	github.com/mattn/go-zglob v0.0.4
+	github.com/peterhellberg/link v1.2.0 // indirect
 	google.golang.org/grpc v1.53.0
 )
-
-replace github.com/atotto/packagecloud => ./

go.sum

@@ -539,6 +539,8 @@ github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuz
 github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
 github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM=
 github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY=
+github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c=
+github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=