From 9933e63c69fe5f7db37fb3f7fd274b9474de1d4f Mon Sep 17 00:00:00 2001 From: "Yann Esposito (Yogsototh)" Date: Tue, 24 Apr 2018 14:54:15 +0200 Subject: [PATCH 1/5] Fix merges and tests, also renamed :identity to :user-id - there were some merge that weren't merged correctly - fix tests - renamed :identity by :user in compojure routes to prevent possible collision of defmethod with the use of ring-jwt-middleware --- src/ctia/bulk/routes.clj | 4 +- src/ctia/bundle/routes.clj | 2 +- src/ctia/entity/casebook.clj | 6 +- src/ctia/entity/feedback.clj | 2 +- src/ctia/http/middleware/auth.clj | 2 +- src/ctia/http/routes/crud.clj | 12 +- src/ctia/observable/routes.clj | 12 +- test/ctia/auth/jwt_test.clj | 4 +- test/ctia/http/routes/bundle_test.clj | 251 ++++++++++++----------- test/ctia/http/routes/judgement_test.clj | 113 +++++----- 10 files changed, 203 insertions(+), 205 deletions(-) diff --git a/src/ctia/bulk/routes.clj b/src/ctia/bulk/routes.clj index 5142465415..68c82a3a03 100644 --- a/src/ctia/bulk/routes.clj +++ b/src/ctia/bulk/routes.clj @@ -169,7 +169,7 @@ :body [bulk NewBulk {:description "a new Bulk object"}] :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "POST many new entities using a single HTTP call" - :identity login + :user-id login :capabilities #{:create-actor :create-attack-pattern :create-campaign @@ -227,7 +227,7 @@ :read-casebook :read-sighting :read-tool} - :identity auth-identity + :user-id auth-identity (let [bulk (into {} (remove (comp empty? second) {:actors actors :attack_patterns attack_patterns diff --git a/src/ctia/bundle/routes.clj b/src/ctia/bundle/routes.clj index 9eff1f972c..f6cc40df13 100644 --- a/src/ctia/bundle/routes.clj +++ b/src/ctia/bundle/routes.clj @@ -355,7 +355,7 @@ nil}] :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "POST many new entities using a single HTTP call" - :identity auth-identity + :user-id auth-identity :capabilities #{:create-actor :create-attack-pattern :create-campaign diff --git a/src/ctia/entity/casebook.clj b/src/ctia/entity/casebook.clj index 45784543f1..5618b78206 100644 --- a/src/ctia/entity/casebook.clj +++ b/src/ctia/entity/casebook.clj @@ -136,7 +136,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "Edit Observables on a casebook" :capabilities :create-casebook - :identity identity + :user-id identity :identity-map identity-map (-> (flows/patch-flow :get-fn #(read-store :casebook @@ -169,7 +169,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "Edit Texts on a casebook" :capabilities :create-casebook - :identity identity + :user-id identity :identity-map identity-map (-> (flows/patch-flow :get-fn #(read-store :casebook @@ -202,7 +202,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "Edit a Bundle on a casebook" :capabilities :create-casebook - :identity identity + :user-id identity :identity-map identity-map (-> (flows/patch-flow :get-fn #(read-store :casebook diff --git a/src/ctia/entity/feedback.clj b/src/ctia/entity/feedback.clj index d146c0a874..c49274b0e3 100644 --- a/src/ctia/entity/feedback.clj +++ b/src/ctia/entity/feedback.clj @@ -61,7 +61,7 @@ :summary "Search Feedback" :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :read-feedback - :identity identity + :user-id identity :identity-map identity-map (-> (read-store :feedback list-records diff --git a/src/ctia/http/middleware/auth.clj b/src/ctia/http/middleware/auth.clj index 149ca6235d..c4d5758300 100644 --- a/src/ctia/http/middleware/auth.clj +++ b/src/ctia/http/middleware/auth.clj @@ -63,7 +63,7 @@ (update acc :lets into [bind-to `(:groups ~'+compojure-api-request+)])) -(defmethod meta/restructure-param :identity [_ bind-to acc] +(defmethod meta/restructure-param :user-id [_ bind-to acc] (update acc :lets into [bind-to `(:identity ~'+compojure-api-request+)])) diff --git a/src/ctia/http/routes/crud.clj b/src/ctia/http/routes/crud.clj index ca1580b1d4..b98a17e777 100644 --- a/src/ctia/http/routes/crud.clj +++ b/src/ctia/http/routes/crud.clj @@ -51,7 +51,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary (format "Adds a new %s" capitalized) :capabilities post-capabilities - :identity identity + :user-id identity :identity-map identity-map (-> (flows/create-flow :entity-type entity @@ -77,7 +77,7 @@ :summary (format "Updates an %s" capitalized) :path-params [id :- s/Str] :capabilities put-capabilities - :identity identity + :user-id identity :identity-map identity-map (-> (flows/update-flow :get-fn #(read-store entity @@ -107,7 +107,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary (format "List %s by external id" capitalized) :capabilities external-id-capabilities - :identity identity + :user-id identity :identity-map identity-map (-> (read-store entity list-records @@ -124,7 +124,7 @@ :summary (format "Search for a %s using a Lucene/ES query string" capitalized) :query [params search-q-params] :capabilities search-capabilities - :identity identity + :user-id identity :identity-map identity-map :header-params [{Authorization :- (s/maybe s/Str) nil}] (-> (query-string-search-store @@ -145,7 +145,7 @@ :query [params get-params] :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities get-capabilities - :identity identity + :user-id identity :identity-map identity-map (if-let [rec (read-store entity read-record @@ -164,7 +164,7 @@ :summary (format "Deletes a %s" capitalized) :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities delete-capabilities - :identity identity + :user-id identity :identity-map identity-map (if (flows/delete-flow :get-fn #(read-store entity diff --git a/src/ctia/observable/routes.clj b/src/ctia/observable/routes.clj index e04b544fac..6217feb303 100644 --- a/src/ctia/observable/routes.clj +++ b/src/ctia/observable/routes.clj @@ -33,7 +33,7 @@ "observable.") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :read-verdict - :identity identity + :user-id identity :identity-map identity-map (or (some-> (read-store :judgement calculate-verdict @@ -53,7 +53,7 @@ :summary "Returns the Judgements associated with the specified observable." :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :list-judgements - :identity identity + :user-id identity :identity-map identity-map (-> (read-store :judgement list-judgements-by-observable @@ -75,7 +75,7 @@ "specified observable based on Judgement relationships.") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities #{:list-judgements :list-relationships} - :identity identity + :user-id identity :identity-map identity-map (paginated-ok (let [http-show (get-in @properties [:ctia :http :show]) @@ -114,7 +114,7 @@ observable_value :- s/Str] :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :list-sightings - :identity identity + :user-id identity :identity-map identity-map :return PartialSightingList :summary "Returns Sightings associated with the specified observable." @@ -138,7 +138,7 @@ "specified observable based on Sighting relationships.") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities #{:list-sightings :list-relationships} - :identity identity + :user-id identity :identity-map identity-map (paginated-ok (let [http-show (get-in @properties [:ctia :http :show]) @@ -179,7 +179,7 @@ "specified observable based on Sighting relationships") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities #{:list-sightings :list-relationships} - :identity identity + :user-id identity :identity-map identity-map (paginated-ok (let [http-show (get-in @properties [:ctia :http :show]) diff --git a/test/ctia/auth/jwt_test.clj b/test/ctia/auth/jwt_test.clj index cd65e73957..e73b4289f2 100644 --- a/test/ctia/auth/jwt_test.clj +++ b/test/ctia/auth/jwt_test.clj @@ -1,6 +1,6 @@ (ns ctia.auth.jwt-test (:require [ctia.auth.jwt :as sut] - [ctia.auth :as auth] + [ctia.auth.capabilities :as caps] [clojure.test :as t :refer [deftest is]] [clojure.set :as set])) @@ -42,7 +42,7 @@ (sut/scope-to-capabilities sut/casebook-root-scope)) "Check the casebook capabilities from the casebook scope") (is (= #{:developer :specify-id :external-id :import-bundle} - (set/difference auth/all-capabilities + (set/difference caps/all-capabilities (sut/scopes-to-capabilities #{sut/entity-root-scope sut/casebook-root-scope}))) "with all scopes you should have most capabilities except some very diff --git a/test/ctia/http/routes/bundle_test.clj b/test/ctia/http/routes/bundle_test.clj index 3853130345..ffc13be041 100644 --- a/test/ctia/http/routes/bundle_test.clj +++ b/test/ctia/http/routes/bundle_test.clj @@ -169,6 +169,7 @@ (map (fn [[k v]] [k (count v)])) (into {}))) + (deftest bundle-import-test (test-for-each-store (fn [] @@ -176,139 +177,139 @@ (whoami-helpers/set-whoami-response "45c1f5e3f05d0" "foouser" "foogroup" - "user")) - (let [indicators [(mk-indicator 0) - (mk-indicator 1)] - sightings [(mk-sighting 0) - (mk-sighting 1)] - relationships (map (fn [idx indicator sighting] - (mk-relationship idx indicator - sighting "indicates")) - (range) - indicators - sightings)] - (testing "Import bundle with all entity types" - (let [new-bundle (deep-dissoc-entity-ids bundle-maximal) - response (post "ctia/bundle/import" - :body new-bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result (:parsed-body response)] - (is (= 200 (:status response))) - (is (= (count-bundle-entities new-bundle) - (count-bundle-result-entities (:results bundle-result) - "created")) - "All entities are created"))) - (testing "Create" - (let [bundle {:type "bundle" - :source "source" - :indicators (set indicators) - :sightings (set sightings) - :relationships (set relationships)} - response (post "ctia/bundle/import" - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result (:parsed-body response)] - (is (= 200 (:status response))) + "user") + (let [indicators [(mk-indicator 0) + (mk-indicator 1)] + sightings [(mk-sighting 0) + (mk-sighting 1)] + relationships (map (fn [idx indicator sighting] + (mk-relationship idx indicator + sighting "indicates")) + (range) + indicators + sightings)] + (testing "Import bundle with all entity types" + (let [new-bundle (deep-dissoc-entity-ids bundle-maximal) + response (post "ctia/bundle/import" + :body new-bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result (:parsed-body response)] + (is (= 200 (:status response))) + (is (= (count-bundle-entities new-bundle) + (count-bundle-result-entities (:results bundle-result) + "created")) + "All entities are created"))) + (testing "Create" + (let [bundle {:type "bundle" + :source "source" + :indicators (set indicators) + :sightings (set sightings) + :relationships (set relationships)} + response (post "ctia/bundle/import" + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result (:parsed-body response)] + (is (= 200 (:status response))) - (is (every? #(= "created" %) - (map :result (:results bundle-result))) - "All entities are created") + (is (every? #(= "created" %) + (map :result (:results bundle-result))) + "All entities are created") - (doseq [entity (concat indicators - sightings - (map #(resolve-ids bundle-result %) - relationships))] - (validate-entity-record - (find-result-by-original-id bundle-result (:id entity)) - entity)))) - (testing "Update" - (let [bundle - {:type "bundle" - :source "source" - :indicators (set (map with-modified-description indicators)) - :sightings (set (map with-modified-description sightings)) - :relationships (set (map with-modified-description relationships))} - response (post "ctia/bundle/import" - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result (:parsed-body response)] - (is (= 200 (:status response))) + (doseq [entity (concat indicators + sightings + (map #(resolve-ids bundle-result %) + relationships))] + (validate-entity-record + (find-result-by-original-id bundle-result (:id entity)) + entity)))) + (testing "Update" + (let [bundle + {:type "bundle" + :source "source" + :indicators (set (map with-modified-description indicators)) + :sightings (set (map with-modified-description sightings)) + :relationships (set (map with-modified-description relationships))} + response (post "ctia/bundle/import" + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result (:parsed-body response)] + (is (= 200 (:status response))) - (is (pos? (count (:results bundle-result)))) + (is (pos? (count (:results bundle-result)))) - (is (every? #(= "exists" %) - (map :result (:results bundle-result))) - "All existing entities are not updated") + (is (every? #(= "exists" %) + (map :result (:results bundle-result))) + "All existing entities are not updated") - (doseq [entity (concat indicators - sightings - (map #(resolve-ids bundle-result %) - relationships))] - (validate-entity-record - (find-result-by-original-id bundle-result (:id entity)) - entity)))) - (testing "Custom external prefix keys" - (let [bundle {:type "bundle" - :source "source" - :indicators (hash-set - (assoc (first indicators) - :external_ids - ["custom-2"]))} - response-create (post "ctia/bundle/import" - :query-params {"external-key-prefixes" "custom-"} - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result-create (:parsed-body response-create) - response-update (post "ctia/bundle/import" - :query-params {"external-key-prefixes" "custom-"} - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result-update (:parsed-body response-update)] - (is (= 200 (:status response-create))) - (is (= 200 (:status response-update))) + (doseq [entity (concat indicators + sightings + (map #(resolve-ids bundle-result %) + relationships))] + (validate-entity-record + (find-result-by-original-id bundle-result (:id entity)) + entity)))) + (testing "Custom external prefix keys" + (let [bundle {:type "bundle" + :source "source" + :indicators (hash-set + (assoc (first indicators) + :external_ids + ["custom-2"]))} + response-create (post "ctia/bundle/import" + :query-params {"external-key-prefixes" "custom-"} + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result-create (:parsed-body response-create) + response-update (post "ctia/bundle/import" + :query-params {"external-key-prefixes" "custom-"} + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result-update (:parsed-body response-update)] + (is (= 200 (:status response-create))) + (is (= 200 (:status response-update))) - (is (pos? (count (:results bundle-result-create)))) - (is (pos? (count (:results bundle-result-update)))) + (is (pos? (count (:results bundle-result-create)))) + (is (pos? (count (:results bundle-result-update)))) - (is (every? #(= "created" %) - (map :result (:results bundle-result-create))) - "All new entities are created") - (is (every? #(= "exists" %) - (map :result (:results bundle-result-update))) - "All existing entities are not updated"))) - (testing "Partial results with errors" - (let [indicator-store-state (-> @stores :indicator first :state) - indexname (:index indicator-store-state)] - (es-index/close! (:conn indicator-store-state) indexname)) + (is (every? #(= "created" %) + (map :result (:results bundle-result-create))) + "All new entities are created") + (is (every? #(= "exists" %) + (map :result (:results bundle-result-update))) + "All existing entities are not updated"))) + (testing "Partial results with errors" + (let [indicator-store-state (-> @stores :indicator first :state) + indexname (:index indicator-store-state)] + (es-index/close! (:conn indicator-store-state) indexname)) - (let [bundle {:type "bundle" - :source "source" - :sightings [(mk-sighting 10) - (mk-sighting 11)] - ;; Remove external_ids to avoid errors - ;; coming from the search operation - :indicators [(dissoc (mk-indicator 10) - :external_ids)]} - response-create (post "ctia/bundle/import" - :body bundle - :headers {"Authorization" "45c1f5e3f05d0"}) - bundle-result-create (:parsed-body response-create)] - (is (= 200 (:status response-create))) - (is (every? #(= "created" %) - (->> (:results bundle-result-create) - (filter #(= "sighting" %)) - (map :result))) - "All valid entities are created") - (doseq [entity (:sightings bundle)] - (validate-entity-record - (find-result-by-original-id bundle-result-create (:id entity)) - entity)) - (let [indicators (filter - #(= :indicator (:type %)) - (:results bundle-result-create))] - (is (not (empty? indicators)) - "The result collection for indicators is not empty") - (is (every? #(contains? % :error) indicators)))))))) + (let [bundle {:type "bundle" + :source "source" + :sightings [(mk-sighting 10) + (mk-sighting 11)] + ;; Remove external_ids to avoid errors + ;; coming from the search operation + :indicators [(dissoc (mk-indicator 10) + :external_ids)]} + response-create (post "ctia/bundle/import" + :body bundle + :headers {"Authorization" "45c1f5e3f05d0"}) + bundle-result-create (:parsed-body response-create)] + (is (= 200 (:status response-create))) + (is (every? #(= "created" %) + (->> (:results bundle-result-create) + (filter #(= "sighting" %)) + (map :result))) + "All valid entities are created") + (doseq [entity (:sightings bundle)] + (validate-entity-record + (find-result-by-original-id bundle-result-create (:id entity)) + entity)) + (let [indicators (filter + #(= :indicator (:type %)) + (:results bundle-result-create))] + (is (not (empty? indicators)) + "The result collection for indicators is not empty") + (is (every? #(contains? % :error) indicators))))))))) (deftest find-by-external-ids-test (test-for-each-store diff --git a/test/ctia/http/routes/judgement_test.clj b/test/ctia/http/routes/judgement_test.clj index c4eb95d809..110766ee8b 100644 --- a/test/ctia/http/routes/judgement_test.clj +++ b/test/ctia/http/routes/judgement_test.clj @@ -195,33 +195,33 @@ :headers {"Authorization" "Bearer 45c1f5e3f05d0"})] (is (= 401 (:status response))))) - (testing "GET /ctia/judgement/:id with JWT Authorization header" - (with-redefs [time/now (constantly (time/date-time 2017 02 16 0 0 0))] - (let [jwt-token "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL3VzZXJcL2VtYWlsIjoiZ2J1aXNzb24rcWFfc2RjX2lyb2hAY2lzY28uY29tIiwiaHR0cHM6XC9cL3NjaGVtYXMuY2lzY28uY29tXC9pcm9oXC9pZGVudGl0eVwvY2xhaW1zXC91c2VyXC9pZHBcL2lkIjoiYW1wIiwiaHR0cHM6XC9cL3NjaGVtYXMuY2lzY28uY29tXC9pcm9oXC9pZGVudGl0eVwvY2xhaW1zXC91c2VyXC9uaWNrIjoiZ2J1aXNzb24rcWFfc2RjX2lyb2hAY2lzY28uY29tIiwiZW1haWwiOiJnYnVpc3NvbitxYV9zZGNfaXJvaEBjaXNjby5jb20iLCJzdWIiOiI1NmJiNWY4Yy1jYzRlLTRlZDMtYTkxYS1jNjYwNDI4N2ZlMzIiLCJpc3MiOiJJUk9IIEF1dGgiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL3Njb3BlcyI6WyJjYXNlYm9vayIsImdsb2JhbC1pbnRlbCIsInByaXZhdGUtaW50ZWwiLCJjb2xsZWN0IiwiZW5yaWNoIiwiaW5zcGVjdCIsImludGVncmF0aW9uIiwiaXJvaC1hdXRoIiwicmVzcG9uc2UiLCJ1aS1zZXR0aW5ncyJdLCJleHAiOjE0ODc3NzI4NTAsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvb2F1dGhcL2NsaWVudFwvbmFtZSI6Imlyb2gtdWkiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL29yZ1wvaWQiOiI2MzQ4OWNmOS01NjFjLTQ5NTgtYTEzZC02ZDg0YjdlZjA5ZDQiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL29yZ1wvbmFtZSI6IklST0ggVGVzdGluZyIsImp0aSI6ImEyNjhhZTdhMy0wOWM5LTQxNDktYjQ5NS1iOThjOGM1ZGU2NjYiLCJuYmYiOjE0ODcxNjc3NTAsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvdXNlclwvaWQiOiI1NmJiNWY4Yy1jYzRlLTRlZDMtYTkxYS1jNjYwNDI4N2ZlMzIiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL29hdXRoXC9jbGllbnRcL2lkIjoiaXJvaC11aSIsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvdmVyc2lvbiI6IjEiLCJpYXQiOjE0ODcxNjgwNTAsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvb2F1dGhcL2tpbmQiOiJzZXNzaW9uLXRva2VuIn0.jl0r3LiL6qOy6DIDZs5NRiQBHlJEzXFXUvKXGPd2PL66xSE0v0Bkc6FD3vPccYxvk-tWBMJX8oiDuAgYt2eRU05blPtzy1yQ-V-zJtxnpuQbDzvVytZvE9n1_8NdvcLa9eXBjUkJ2FsXAIguXpVDIbR3zs9MkjfyrsKeVCmhC3QTehj55Rf-WINeTq0UflIyoZqfK5Mewl-DBwbvTRjTIRJpNPhjErJ0ypHNXzTKM-nVljSRhrfpoBYpPxQSQVTedWIA2Sks4fBvEwdeE60aBRK1HeTps0G1h3RXPYu7q1I5ti9a2axiQtRLA11CxoOvMmnjyWkffi5vyrFKqZ7muQ" - {status :status - judgement :parsed-body - :as response} - (get (str "ctia/judgement/" (:short-id judgement-id)) - :headers {"Authorization" (str "Bearer " jwt-token)})] - (is (= 200 (:status response))) - (is (= {:id (id/long-id judgement-id) - :type "judgement" - :observable {:value "1.2.3.4" - :type "ip"} - :external_ids ["http://ex.tld/ctia/judgement/judgement-123" - "http://ex.tld/ctia/judgement/judgement-456"] - :disposition 2 - :disposition_name "Malicious" - :priority 100 - :severity "High" - :confidence "Low" - :source "test" - :tlp "green" - :schema_version schema-version - :reason "This is a bad IP address that talked to some evil servers" - :valid_time {:start_time #inst "2016-02-11T00:40:48.212-00:00" - :end_time #inst "2525-01-01T00:00:00.000-00:00"}} - judgement)))))))) + (testing "GET /ctia/judgement/:id with JWT Authorization header" + (with-redefs [time/now (constantly (time/date-time 2017 02 16 0 0 0))] + (let [jwt-token "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL3VzZXJcL2VtYWlsIjoiZ2J1aXNzb24rcWFfc2RjX2lyb2hAY2lzY28uY29tIiwiaHR0cHM6XC9cL3NjaGVtYXMuY2lzY28uY29tXC9pcm9oXC9pZGVudGl0eVwvY2xhaW1zXC91c2VyXC9pZHBcL2lkIjoiYW1wIiwiaHR0cHM6XC9cL3NjaGVtYXMuY2lzY28uY29tXC9pcm9oXC9pZGVudGl0eVwvY2xhaW1zXC91c2VyXC9uaWNrIjoiZ2J1aXNzb24rcWFfc2RjX2lyb2hAY2lzY28uY29tIiwiZW1haWwiOiJnYnVpc3NvbitxYV9zZGNfaXJvaEBjaXNjby5jb20iLCJzdWIiOiI1NmJiNWY4Yy1jYzRlLTRlZDMtYTkxYS1jNjYwNDI4N2ZlMzIiLCJpc3MiOiJJUk9IIEF1dGgiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL3Njb3BlcyI6WyJjYXNlYm9vayIsImdsb2JhbC1pbnRlbCIsInByaXZhdGUtaW50ZWwiLCJjb2xsZWN0IiwiZW5yaWNoIiwiaW5zcGVjdCIsImludGVncmF0aW9uIiwiaXJvaC1hdXRoIiwicmVzcG9uc2UiLCJ1aS1zZXR0aW5ncyJdLCJleHAiOjE0ODc3NzI4NTAsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvb2F1dGhcL2NsaWVudFwvbmFtZSI6Imlyb2gtdWkiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL29yZ1wvaWQiOiI2MzQ4OWNmOS01NjFjLTQ5NTgtYTEzZC02ZDg0YjdlZjA5ZDQiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL29yZ1wvbmFtZSI6IklST0ggVGVzdGluZyIsImp0aSI6ImEyNjhhZTdhMy0wOWM5LTQxNDktYjQ5NS1iOThjOGM1ZGU2NjYiLCJuYmYiOjE0ODcxNjc3NTAsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvdXNlclwvaWQiOiI1NmJiNWY4Yy1jYzRlLTRlZDMtYTkxYS1jNjYwNDI4N2ZlMzIiLCJodHRwczpcL1wvc2NoZW1hcy5jaXNjby5jb21cL2lyb2hcL2lkZW50aXR5XC9jbGFpbXNcL29hdXRoXC9jbGllbnRcL2lkIjoiaXJvaC11aSIsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvdmVyc2lvbiI6IjEiLCJpYXQiOjE0ODcxNjgwNTAsImh0dHBzOlwvXC9zY2hlbWFzLmNpc2NvLmNvbVwvaXJvaFwvaWRlbnRpdHlcL2NsYWltc1wvb2F1dGhcL2tpbmQiOiJzZXNzaW9uLXRva2VuIn0.jl0r3LiL6qOy6DIDZs5NRiQBHlJEzXFXUvKXGPd2PL66xSE0v0Bkc6FD3vPccYxvk-tWBMJX8oiDuAgYt2eRU05blPtzy1yQ-V-zJtxnpuQbDzvVytZvE9n1_8NdvcLa9eXBjUkJ2FsXAIguXpVDIbR3zs9MkjfyrsKeVCmhC3QTehj55Rf-WINeTq0UflIyoZqfK5Mewl-DBwbvTRjTIRJpNPhjErJ0ypHNXzTKM-nVljSRhrfpoBYpPxQSQVTedWIA2Sks4fBvEwdeE60aBRK1HeTps0G1h3RXPYu7q1I5ti9a2axiQtRLA11CxoOvMmnjyWkffi5vyrFKqZ7muQ" + {status :status + judgement :parsed-body + :as response} + (get (str "ctia/judgement/" (:short-id judgement-id)) + :headers {"Authorization" (str "Bearer " jwt-token)})] + (is (= 200 (:status response))) + (is (= {:id (id/long-id judgement-id) + :type "judgement" + :observable {:value "1.2.3.4" + :type "ip"} + :external_ids ["http://ex.tld/ctia/judgement/judgement-123" + "http://ex.tld/ctia/judgement/judgement-456"] + :disposition 2 + :disposition_name "Malicious" + :priority 100 + :severity "High" + :confidence "Low" + :source "test" + :tlp "green" + :schema_version schema-version + :reason "This is a bad IP address that talked to some evil servers" + :valid_time {:start_time #inst "2016-02-11T00:40:48.212-00:00" + :end_time #inst "2525-01-01T00:00:00.000-00:00"}} + judgement)))))))))) (deftest test-judgement-routes-for-dispositon-determination (test-for-each-store @@ -324,35 +324,34 @@ (dissoc judgement :id))))) -<<<<<<< HEAD - (testing "POST a judgement with mismatching disposition/disposition_name" - (let [{status :status - judgement :parsed-body} - (post "ctia/judgement" - :body {:observable {:value "1.2.3.4" - :type "ip"} - :disposition 1 - :disposition_name "Unknown" - :source "test" - :priority 100 - :severity "High" - :confidence "Low" - :valid_time {:start_time "2016-02-11T00:40:48.212-00:00"}} - :headers {"Authorization" "45c1f5e3f05d0"})] - (is (= 400 status)) - (is (= - {:error "Mismatching :dispostion and dispositon_name for judgement", - :judgement {:observable {:value "1.2.3.4" - :type "ip"} - :disposition 1 - :disposition_name "Unknown" - :source "test" - :priority 100 - :severity "High" - :confidence "Low" - :valid_time {:start_time #inst "2016-02-11T00:40:48.212-00:00"}}} - judgement))))) -======= + (testing "POST a judgement with mismatching disposition/disposition_name" + (let [{status :status + judgement :parsed-body} + (post "ctia/judgement" + :body {:observable {:value "1.2.3.4" + :type "ip"} + :disposition 1 + :disposition_name "Unknown" + :source "test" + :priority 100 + :severity "High" + :confidence "Low" + :valid_time {:start_time "2016-02-11T00:40:48.212-00:00"}} + :headers {"Authorization" "45c1f5e3f05d0"})] + (is (= 400 status)) + (is (= + {:error "Mismatching :dispostion and dispositon_name for judgement", + :judgement {:observable {:value "1.2.3.4" + :type "ip"} + :disposition 1 + :disposition_name "Unknown" + :source "test" + :priority 100 + :severity "High" + :confidence "Low" + :valid_time {:start_time #inst "2016-02-11T00:40:48.212-00:00"}}} + judgement)))) + (testing "POST a judgement with mismatching disposition/disposition_name" (let [{status :status judgement :parsed-body} @@ -380,8 +379,6 @@ :confidence "Low" :valid_time {:start_time #inst "2016-02-11T00:40:48.212-00:00"}}} judgement))))))) ->>>>>>> remove deftest-for-each-store - (deftest test-judgement-pagination-field-selection (test-for-each-store From d0d0966d97c334720fd53e6b383c068f0208f276 Mon Sep 17 00:00:00 2001 From: "Yann Esposito (Yogsototh)" Date: Tue, 24 Apr 2018 16:05:21 +0200 Subject: [PATCH 2/5] replaced '(remove nil? (map ...))` by `(keep ...)` --- src/ctia/auth/capabilities.clj | 5 ++--- src/ctia/bundle/routes.clj | 2 +- src/ctia/flows/crud.clj | 14 ++++++-------- test/ctia/test_helpers/pagination.clj | 6 ++---- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/ctia/auth/capabilities.clj b/src/ctia/auth/capabilities.clj index ba10187317..98bda4c6cf 100644 --- a/src/ctia/auth/capabilities.clj +++ b/src/ctia/auth/capabilities.clj @@ -10,9 +10,8 @@ :specify-id :external-id :import-bundle} - (remove nil? - (map (fn [[_ entity]] - (:capabilities entity)) entities)))) + (keep (fn [[_ entity]] + (:capabilities entity)) entities))) (def default-capabilities {:user diff --git a/src/ctia/bundle/routes.clj b/src/ctia/bundle/routes.clj index f6cc40df13..8e48d78e06 100644 --- a/src/ctia/bundle/routes.clj +++ b/src/ctia/bundle/routes.clj @@ -140,7 +140,7 @@ (defn find-by-external-ids [import-data entity-type auth-identity] - (let [external-ids (remove nil? (map :external_id import-data))] + (let [external-ids (keep :external_id import-data)] (log/debugf "Searching %s matching these external_ids %s" entity-type (pr-str external-ids)) diff --git a/src/ctia/flows/crud.clj b/src/ctia/flows/crud.clj index fbf65b9d24..cf290cf300 100644 --- a/src/ctia/flows/crud.clj +++ b/src/ctia/flows/crud.clj @@ -113,10 +113,9 @@ :as fm} :- FlowMap] (let [newtempids (->> entities - (map (fn [{:keys [id]}] - (when (and id (re-matches id/transient-id-re id)) - [id (make-id entity-type)]))) - (remove nil?) + (keep (fn [{:keys [id]}] + (when (and id (re-matches id/transient-id-re id)) + [id (make-id entity-type)]))) (into {}))] (update fm :tempids (fnil into {}) newtempids))) @@ -268,10 +267,9 @@ "Converts IDs in the tempids mapping table from short to long format" [tempids short-to-long-map] (->> tempids - (map (fn [[tempid short-id]] - (when-let [long-id (get short-to-long-map short-id)] - [tempid long-id]))) - (remove nil?) + (keep (fn [[tempid short-id]] + (when-let [long-id (get short-to-long-map short-id)] + [tempid long-id]))) (into {}))) (s/defn ^:private apply-long-id-fn :- FlowMap diff --git a/test/ctia/test_helpers/pagination.clj b/test/ctia/test_helpers/pagination.clj index 4f8cd8b32c..c43efd46fe 100644 --- a/test/ctia/test_helpers/pagination.clj +++ b/test/ctia/test_helpers/pagination.clj @@ -84,15 +84,13 @@ (let [manually-sorted (->> (get route :headers headers) :parsed-body (sort-by field) - (map field) - (remove nil?)) + (keep field)) route-sorted (->> (get route :headers headers :query-params {:sort_by (name field) :sort_order "asc"}) :parsed-body - (map field) - (remove nil?))] + (keep field))] (is (deep= manually-sorted route-sorted) field))) From e15fbf75c8ca3db88be6ccf2621b1cba6fc65775 Mon Sep 17 00:00:00 2001 From: "Yann Esposito (Yogsototh)" Date: Wed, 25 Apr 2018 11:22:16 +0200 Subject: [PATCH 3/5] use :auth-identity instead of :identity --- src/ctia/bulk/routes.clj | 4 ++-- src/ctia/bundle/routes.clj | 2 +- src/ctia/entity/casebook.clj | 6 +++--- src/ctia/entity/feedback.clj | 2 +- src/ctia/http/middleware/auth.clj | 2 +- src/ctia/http/routes/crud.clj | 12 ++++++------ src/ctia/observable/routes.clj | 12 ++++++------ 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/ctia/bulk/routes.clj b/src/ctia/bulk/routes.clj index 68c82a3a03..237d1d001c 100644 --- a/src/ctia/bulk/routes.clj +++ b/src/ctia/bulk/routes.clj @@ -169,7 +169,7 @@ :body [bulk NewBulk {:description "a new Bulk object"}] :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "POST many new entities using a single HTTP call" - :user-id login + :auth-identity login :capabilities #{:create-actor :create-attack-pattern :create-campaign @@ -227,7 +227,7 @@ :read-casebook :read-sighting :read-tool} - :user-id auth-identity + :auth-identity auth-identity (let [bulk (into {} (remove (comp empty? second) {:actors actors :attack_patterns attack_patterns diff --git a/src/ctia/bundle/routes.clj b/src/ctia/bundle/routes.clj index 8e48d78e06..e64f6049ab 100644 --- a/src/ctia/bundle/routes.clj +++ b/src/ctia/bundle/routes.clj @@ -355,7 +355,7 @@ nil}] :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "POST many new entities using a single HTTP call" - :user-id auth-identity + :auth-identity auth-identity :capabilities #{:create-actor :create-attack-pattern :create-campaign diff --git a/src/ctia/entity/casebook.clj b/src/ctia/entity/casebook.clj index 5618b78206..81674b20e3 100644 --- a/src/ctia/entity/casebook.clj +++ b/src/ctia/entity/casebook.clj @@ -136,7 +136,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "Edit Observables on a casebook" :capabilities :create-casebook - :user-id identity + :auth-identity identity :identity-map identity-map (-> (flows/patch-flow :get-fn #(read-store :casebook @@ -169,7 +169,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "Edit Texts on a casebook" :capabilities :create-casebook - :user-id identity + :auth-identity identity :identity-map identity-map (-> (flows/patch-flow :get-fn #(read-store :casebook @@ -202,7 +202,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary "Edit a Bundle on a casebook" :capabilities :create-casebook - :user-id identity + :auth-identity identity :identity-map identity-map (-> (flows/patch-flow :get-fn #(read-store :casebook diff --git a/src/ctia/entity/feedback.clj b/src/ctia/entity/feedback.clj index c49274b0e3..0b79ef01e0 100644 --- a/src/ctia/entity/feedback.clj +++ b/src/ctia/entity/feedback.clj @@ -61,7 +61,7 @@ :summary "Search Feedback" :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :read-feedback - :user-id identity + :auth-identity identity :identity-map identity-map (-> (read-store :feedback list-records diff --git a/src/ctia/http/middleware/auth.clj b/src/ctia/http/middleware/auth.clj index c4d5758300..b7c355ccdd 100644 --- a/src/ctia/http/middleware/auth.clj +++ b/src/ctia/http/middleware/auth.clj @@ -63,7 +63,7 @@ (update acc :lets into [bind-to `(:groups ~'+compojure-api-request+)])) -(defmethod meta/restructure-param :user-id [_ bind-to acc] +(defmethod meta/restructure-param :auth-identity [_ bind-to acc] (update acc :lets into [bind-to `(:identity ~'+compojure-api-request+)])) diff --git a/src/ctia/http/routes/crud.clj b/src/ctia/http/routes/crud.clj index b98a17e777..96aeaa42ba 100644 --- a/src/ctia/http/routes/crud.clj +++ b/src/ctia/http/routes/crud.clj @@ -51,7 +51,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary (format "Adds a new %s" capitalized) :capabilities post-capabilities - :user-id identity + :auth-identity identity :identity-map identity-map (-> (flows/create-flow :entity-type entity @@ -77,7 +77,7 @@ :summary (format "Updates an %s" capitalized) :path-params [id :- s/Str] :capabilities put-capabilities - :user-id identity + :auth-identity identity :identity-map identity-map (-> (flows/update-flow :get-fn #(read-store entity @@ -107,7 +107,7 @@ :header-params [{Authorization :- (s/maybe s/Str) nil}] :summary (format "List %s by external id" capitalized) :capabilities external-id-capabilities - :user-id identity + :auth-identity identity :identity-map identity-map (-> (read-store entity list-records @@ -124,7 +124,7 @@ :summary (format "Search for a %s using a Lucene/ES query string" capitalized) :query [params search-q-params] :capabilities search-capabilities - :user-id identity + :auth-identity identity :identity-map identity-map :header-params [{Authorization :- (s/maybe s/Str) nil}] (-> (query-string-search-store @@ -145,7 +145,7 @@ :query [params get-params] :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities get-capabilities - :user-id identity + :auth-identity identity :identity-map identity-map (if-let [rec (read-store entity read-record @@ -164,7 +164,7 @@ :summary (format "Deletes a %s" capitalized) :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities delete-capabilities - :user-id identity + :auth-identity identity :identity-map identity-map (if (flows/delete-flow :get-fn #(read-store entity diff --git a/src/ctia/observable/routes.clj b/src/ctia/observable/routes.clj index 6217feb303..bf0d899819 100644 --- a/src/ctia/observable/routes.clj +++ b/src/ctia/observable/routes.clj @@ -33,7 +33,7 @@ "observable.") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :read-verdict - :user-id identity + :auth-identity identity :identity-map identity-map (or (some-> (read-store :judgement calculate-verdict @@ -53,7 +53,7 @@ :summary "Returns the Judgements associated with the specified observable." :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :list-judgements - :user-id identity + :auth-identity identity :identity-map identity-map (-> (read-store :judgement list-judgements-by-observable @@ -75,7 +75,7 @@ "specified observable based on Judgement relationships.") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities #{:list-judgements :list-relationships} - :user-id identity + :auth-identity identity :identity-map identity-map (paginated-ok (let [http-show (get-in @properties [:ctia :http :show]) @@ -114,7 +114,7 @@ observable_value :- s/Str] :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities :list-sightings - :user-id identity + :auth-identity identity :identity-map identity-map :return PartialSightingList :summary "Returns Sightings associated with the specified observable." @@ -138,7 +138,7 @@ "specified observable based on Sighting relationships.") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities #{:list-sightings :list-relationships} - :user-id identity + :auth-identity identity :identity-map identity-map (paginated-ok (let [http-show (get-in @properties [:ctia :http :show]) @@ -179,7 +179,7 @@ "specified observable based on Sighting relationships") :header-params [{Authorization :- (s/maybe s/Str) nil}] :capabilities #{:list-sightings :list-relationships} - :user-id identity + :auth-identity identity :identity-map identity-map (paginated-ok (let [http-show (get-in @properties [:ctia :http :show]) From 08c1c749271c36d70cb4f307687adeead3041832 Mon Sep 17 00:00:00 2001 From: "Yann Esposito (Yogsototh)" Date: Wed, 25 Apr 2018 11:39:27 +0200 Subject: [PATCH 4/5] fix the import-bundle capability from scopes --- src/ctia/auth/jwt.clj | 13 ++++++++++--- test/ctia/auth/jwt_test.clj | 12 +++++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/ctia/auth/jwt.clj b/src/ctia/auth/jwt.clj index 1b7b005f35..2fdc53e07a 100644 --- a/src/ctia/auth/jwt.clj +++ b/src/ctia/auth/jwt.clj @@ -67,12 +67,19 @@ [scope-repr] (case (count (:path scope-repr)) ;; example: ["private-intel" "sighting"] (for private-intel/sighting scope) - 2 (gen-capabilities-for-entity-and-accesses (second (:path scope-repr)) - (:access scope-repr)) + 2 (condp = (second (:path scope-repr)) + "import-bundle" (if (contains? (:access scope-repr) :write) + #{:import-bundle} + #{}) + (gen-capabilities-for-entity-and-accesses (second (:path scope-repr)) + (:access scope-repr))) ;; typically: ["private-intel"] 1 (->> entities (map #(gen-capabilities-for-entity-and-accesses % (:access scope-repr))) - unionize) + unionize + (set/union (if (contains? (:access scope-repr) :write) + #{:import-bundle} + #{}))) #{})) (defn gen-casebook-capabilities diff --git a/test/ctia/auth/jwt_test.clj b/test/ctia/auth/jwt_test.clj index e73b4289f2..a39f98fd00 100644 --- a/test/ctia/auth/jwt_test.clj +++ b/test/ctia/auth/jwt_test.clj @@ -41,12 +41,22 @@ :delete-casebook} (sut/scope-to-capabilities sut/casebook-root-scope)) "Check the casebook capabilities from the casebook scope") - (is (= #{:developer :specify-id :external-id :import-bundle} + (is (= #{:developer :specify-id :external-id} (set/difference caps/all-capabilities (sut/scopes-to-capabilities #{sut/entity-root-scope sut/casebook-root-scope}))) "with all scopes you should have most capabilities except some very specific ones") + (is (= #{:import-bundle} + (sut/scopes-to-capabilities + #{(str sut/entity-root-scope "/import-bundle")}))) + (is (= #{} + (sut/scopes-to-capabilities + #{(str sut/entity-root-scope "/import-bundle:read")}))) + (is (contains? (sut/scopes-to-capabilities #{(str sut/entity-root-scope ":write")}) + :import-bundle)) + (is (not (contains? (sut/scopes-to-capabilities #{(str sut/entity-root-scope ":read")}) + :import-bundle))) (is (= #{:read-sighting :list-sightings :search-sighting :create-sighting :delete-sighting} (sut/scopes-to-capabilities From 9d2f7f3942f12ffc8dfba71b5115e5956cbd913a Mon Sep 17 00:00:00 2001 From: "Yann Esposito (Yogsototh)" Date: Wed, 25 Apr 2018 11:59:58 +0200 Subject: [PATCH 5/5] Add a CONTRIBUTING.md file --- CONTRIBUTING.org | 194 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 CONTRIBUTING.org diff --git a/CONTRIBUTING.org b/CONTRIBUTING.org new file mode 100644 index 0000000000..d7600203bb --- /dev/null +++ b/CONTRIBUTING.org @@ -0,0 +1,194 @@ +#+Title: Contributing + +* Introduction + +This document introduce how to minimize the friction between contributors. +It is mainly written for developpers. + +* Coding Style + +** Syntax + +The code should follow the following guidelines: + +https://github.com/bbatsov/clojure-style-guide + +Mainly it states that the syntax should be the one provided by default with emacs. +Most (every) developer use emacs right now to contribute. + +** Guidelines + +The motivations for these guidelines are code readability and also management of +code complexity: + +- Select meaningful names for functions and variables. Avoid acronyms that + aren't commonly known. +- Prefer rewriting code to make it clear over adding comments. If that cannot be + achieved, add comments. +- Limit the number of functions invoked in a single line of code to five. +- Limit the nested depth of a function definition to four. In order to achieve + this it may be necessary to introduce local variables to hold intermediate + results or extract code into additional function definitions. +- Only use anonymous functions for short function definitions that fit + comfortably on a single line. Otherwise create a private, named function. +- Do not write anonymous functions that have arguments that are anonymous + functions. +- Make an effort to limit the depth of the code. Mainly, you should reach a lib + or core function as soon as possible while following the functions of a + namespace. Other member of the team should not need to open more than three + namespaces at a time to reach all the functions necessary to understand your + code. +- Do not use loop/recur unless you can't use reduce. Do not use reduce unless + you can't use a combination/threading of functions. +- Never use ~defmacro~ unless you really can't use a function and every other + member of the team couldn't find a better way. +- Never use ~defmethod~. +- Only use ~defprotocol~ for declaring TK services. Never use it unless you + can't do otherwise. Most of the time using ~case~ and/or ~cond~ will be preferable. +- Try to avoid the use of ~core.async~ there is certainly an easier, simpler and + less bug prone way to do what you intend to. + +** Backward Compatibility + +We work in a production environment. +Once a code is in production, people are using in some way, we can't break it. + +Try to make your code upgrade proof. +Inspired from (Spec-ulation talk of Rich Hickey +https://www.youtube.com/watch?v=oyLBGkS5ICk). +Ask yourself the questions: + +- "Will we be able to upgrade without downtime?" +- "Will it be easy to put that new code in production without breaking the service?" +- "Will that change imply to run a data migration in production?" +- "Will that change break the workflow of any client?" +- "How easy will it be to upgrade the configuration?" +- "What will occur if the 3rd party change slightly their data format?". + +For example: notice that adding new fields to the configuration is not as +expensive as modifying it or removing a key, because it will force that the +deploiement of the conf and of the code to be synchronized. While if you can add +new fields, you can upgrade the configuration before upgrading the code in +production without any downtime. + +- For most spec/schemas, make them as permissive as possible. + For example ~(defschema Conf {:port s/Int})~ is not as permissive as + as ~(defschema Conf {:port s/Int s/Any s/Any})~. + Because if the configuration change we would be able to add new fields + to the configuration without breaking the running version. +- Only add new fields, new values. + Because modifying the type of a value or removing a field will + break between different version of the code. + #+BEGIN_SRC + OK: (s/enum :a :b) => (s/enum :a :b :c) + NOT OK: (s/enum :a :b :c) => (s/enum :a :b) + + OK: {:a s/Int} => {:a s/Int :b s/Bool} + NOT OK: {:a s/Int} => {:a s/Bool} + #+END_SRC + + +* Git / Deploiement Workflow + +In our team, GitHub is currently used both as a developer tool and as a task +management tool. This document only focus about the developer tool aspect. + +** Git / Github + +Our workflow is close to the Gitlab flow (see: +https://about.gitlab.com/2014/09/29/gitlab-flow/) + +*** Init your env + +1. Clone the iroh repo in github +2. Your master branch should be the one from your own repository (cloned from threatgrid one) +3. The repository is generally named ~tg~ (or ~motherboard~) +4. You should sync your master branch with ~tg~ time to time with the ~git synctg~ alias + #+BEGIN_SRC + [alias] + synctg = "!oldbr=`git rev-parse --abbrev-ref HEAD` ; git checkout master && git fetch --all && git rebase tg/master && git push ; git checkout $oldbr" + #+END_SRC + +You can (should) also add the repository of other contributors to your remote repos. + +#+BEGIN_SRC +> git remote -v +gbuisson git@github.com:gbuisson/iroh.git (fetch) +gbuisson git@github.com:gbuisson/iroh.git (push) +msprunck git@github.com:msprunck/iroh.git (fetch) +msprunck git@github.com:msprunck/iroh.git (push) +origin git@github.com:YOU/iroh.git (fetch) +origin git@github.com:YOU/iroh.git (push) +tg git@github.com:threatgrid/iroh.git (fetch) +tg git@github.com:threatgrid/iroh.git (push) +yogsototh git@github.com:yogsototh/iroh.git (fetch) +yogsototh git@github.com:yogsototh/iroh.git (push) +#+END_SRC + +You can test everything is working correctly by doing: + +#+BEGIN_SRC +> docker-compose -f containers/dev/docker-compose.yml up -d +> lein test +> lein run +#+END_SRC + + +*** Adding a new feature / fixing an issue + +1. You should have an open issue in github with number ~#XXX~. +2. ~git checkout -b issue-XXX-issue-short-description~ +3. work... ~git commit~ many times +4. Optionally you can clean up your git log history with ~git rebase -i master~, or ~git synctg~ +5. Test locally ~./build/compile.sh && ./build/run-tests.sh && lein tk~ +6. ~git push -u~ will push and create the branch on github +7. Open a PR. In the PR reference the issue by writing (~Close #XXX~ or + ~Connects to #XXX~). If the issue also depend from an Epic please reference + it as well ~Connects to Epic #EEE~). +8. Make changes according to PR feedbacks +9. Either use the =Squash & Merge button= in github or manually rebase. Please + take the time to write a good commit message. For example be inspired by + https://chris.beams.io/posts/git-commit/ + +*** Deploiement + +This part doesn't concern contributor external to the core team. + +We currently have three environments. + + +| Integration (INT) | https://intel.int.iroh.site | master | +| Integration (INT) | https://private.intel.int.iroh.site | master | +| TEST | https://intel.test.iroh.site | rel-X.X.X | +| TEST | https://private.intel.test.iroh.site | rel-X.X.X | +| PROD | https://intel.amp.cisco.com | rel-X.X.X | +| PROD | https://private.intel.amp.cisco.com | rel-X.X.X | + +There could be two cases: + +**** Classical: all features from INT to go up to PROD + +That's the easiest case. QA works on TEST environment and file bugs. + +If a bug is found by QA. Make the PR for Int, and tag it (with a github label, +not git tags) with the release version accordingly. + +**** Release Workflow: some feature won't go up from INT to PROD + +In that case, a branch will be created that won't contains some commits of master. + +If a bug is found by QA. Make the PR from the rel-X.X.X branch. + +And also you should (most of the time) use the same branch to make another PR +directly from INT or manually cherry-pick the PR from rel-X.X.X to INT. That +work of bringing back a fix from release down to INT SHOULD NOT be done by QA. + +We should NEVER make any commit goes down. +Only from INT to TEST and from TEST to PROD or directly from TEST to PROD. +This is why it is your duty not to forget to make two PRs, one to fix TEST the +over one to fix INT. + +**** Configuration Modification + +Any change that need a configuration change must be handled in the ~tenzin~ +repository. The detail about how to do that are out of the scope of this document.