From 1442316eec9b78020fac3585f3f07d953b3b174f Mon Sep 17 00:00:00 2001 From: Julien Perret Date: Tue, 26 Jun 2018 08:55:43 +0200 Subject: [PATCH 1/4] Adding proxy handling capabilities --- .../main/scala/gridscale/http/package.scala | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/http/src/main/scala/gridscale/http/package.scala b/http/src/main/scala/gridscale/http/package.scala index 6ba2b169..305b83ad 100644 --- a/http/src/main/scala/gridscale/http/package.scala +++ b/http/src/main/scala/gridscale/http/package.scala @@ -9,9 +9,10 @@ import java.security.cert.{ Certificate, CertificateFactory } import effectaside._ import org.apache.commons.codec.binary -import org.apache.http.{ HttpEntity, client } +import org.apache.http.{ HttpEntity, HttpHost, client } import org.apache.http.client.methods import org.apache.http.entity.InputStreamEntity +import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.message.BasicHttpRequest import sun.security.provider.X509Factory @@ -76,14 +77,16 @@ package object http { case class Head(headers: Headers = Seq.empty) extends HTTPMethod case class Move(to: String, headers: Headers = Seq.empty) extends HTTPMethod + case class HTTPProxy(hostname: String, port: Int) + object HTTP { def apply() = Effect(new HTTP()) - def client(server: Server) = + def client(server: Server, proxy: Option[HTTPProxy] = None) = server match { - case s: HTTPServer ⇒ httpClient(s.timeout) - case s: HTTPSServer ⇒ HTTPS.newClient(s.socketFactory, s.timeout) + case s: HTTPServer ⇒ httpClient(s.timeout, proxy) + case s: HTTPSServer ⇒ HTTPS.newClient(s.socketFactory, s.timeout, proxy) } def requestConfig(timeout: Time) = @@ -93,7 +96,7 @@ package object http { .setConnectionRequestTimeout(timeout.toMillis.toInt) .build() - def httpClient(timeout: Time) = { + def httpClient(timeout: Time, proxy: Option[HTTPProxy]) = { def connectionManager(timeout: Time) = { val client = new BasicHttpClientConnectionManager() val socketConfig = SocketConfig.custom().setSoTimeout(timeout.toMillis.toInt).build() @@ -101,11 +104,17 @@ package object http { client } + def addProxy(httpClientBuilder: HttpClientBuilder) = proxy match { + case Some(p) ⇒ httpClientBuilder.setProxy(new HttpHost(p.hostname, p.port)) + case None ⇒ httpClientBuilder + } + def newClient(timeout: Time) = - HttpClients.custom(). - //setRedirectStrategy(redirectStrategy). - setConnectionManager(connectionManager(timeout)). - setDefaultRequestConfig(requestConfig(timeout)).build() + addProxy( + HttpClients.custom(). + //setRedirectStrategy(redirectStrategy). + setConnectionManager(connectionManager(timeout)). + setDefaultRequestConfig(requestConfig(timeout))).build() newClient(timeout) } @@ -220,7 +229,7 @@ package object http { def apply(url: String, timeout: Time = 1 minutes, bufferSize: Information = 64 kilobytes) = new HTTPServer(new URI(url), timeout, bufferSize) } - + case class HTTPServer(url: URI, timeout: Time, bufferSize: Information) extends Server object HTTPSServer { @@ -380,11 +389,15 @@ package object http { client } - def newClient(factory: SSLSocketFactory, timeout: Time) = - HttpClients.custom(). + def newClient(factory: SSLSocketFactory, timeout: Time, proxy: Option[HTTPProxy]) = { + def addProxy(httpClientBuilder: HttpClientBuilder) = proxy match { + case Some(p) ⇒ httpClientBuilder.setProxy(new HttpHost(p.hostname, p.port)) + case None ⇒ httpClientBuilder + } + addProxy(HttpClients.custom(). setConnectionManager(connectionManager(factory(timeout), timeout)). - setDefaultRequestConfig(HTTP.requestConfig(timeout)).build() - + setDefaultRequestConfig(HTTP.requestConfig(timeout))).build() + } def readPem(pem: java.io.File)(implicit fileSystem: Effect[FileSystem]) = { val content = fileSystem().readStream(pem)(is ⇒ Source.fromInputStream(is).mkString) val stripped = content.replaceAll(X509Factory.BEGIN_CERT, "").replaceAll(X509Factory.END_CERT, "") From 4bbd94e5c5eea5314d777a49d005c02189430b04 Mon Sep 17 00:00:00 2001 From: Julien Perret Date: Tue, 26 Jun 2018 08:57:51 +0200 Subject: [PATCH 2/4] Adding proxy handling capabilities --- http/src/main/scala/gridscale/http/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/scala/gridscale/http/package.scala b/http/src/main/scala/gridscale/http/package.scala index 305b83ad..7df07918 100644 --- a/http/src/main/scala/gridscale/http/package.scala +++ b/http/src/main/scala/gridscale/http/package.scala @@ -229,7 +229,7 @@ package object http { def apply(url: String, timeout: Time = 1 minutes, bufferSize: Information = 64 kilobytes) = new HTTPServer(new URI(url), timeout, bufferSize) } - + case class HTTPServer(url: URI, timeout: Time, bufferSize: Information) extends Server object HTTPSServer { From 9ee842d3a8ab2d6e00c0c002c19dbeb747680d61 Mon Sep 17 00:00:00 2001 From: Julien Perret Date: Tue, 26 Jun 2018 09:31:57 +0200 Subject: [PATCH 3/4] Refactoring of the addProxy function --- .../main/scala/gridscale/http/package.scala | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/http/src/main/scala/gridscale/http/package.scala b/http/src/main/scala/gridscale/http/package.scala index 7df07918..69273658 100644 --- a/http/src/main/scala/gridscale/http/package.scala +++ b/http/src/main/scala/gridscale/http/package.scala @@ -79,6 +79,11 @@ package object http { case class HTTPProxy(hostname: String, port: Int) + def addProxy(httpClientBuilder: HttpClientBuilder, proxy: Option[HTTPProxy]) = proxy match { + case Some(p) ⇒ httpClientBuilder.setProxy(new HttpHost(p.hostname, p.port)) + case None ⇒ httpClientBuilder + } + object HTTP { def apply() = Effect(new HTTP()) @@ -104,17 +109,12 @@ package object http { client } - def addProxy(httpClientBuilder: HttpClientBuilder) = proxy match { - case Some(p) ⇒ httpClientBuilder.setProxy(new HttpHost(p.hostname, p.port)) - case None ⇒ httpClientBuilder - } - def newClient(timeout: Time) = addProxy( HttpClients.custom(). //setRedirectStrategy(redirectStrategy). setConnectionManager(connectionManager(timeout)). - setDefaultRequestConfig(requestConfig(timeout))).build() + setDefaultRequestConfig(requestConfig(timeout)), proxy).build() newClient(timeout) } @@ -390,13 +390,9 @@ package object http { } def newClient(factory: SSLSocketFactory, timeout: Time, proxy: Option[HTTPProxy]) = { - def addProxy(httpClientBuilder: HttpClientBuilder) = proxy match { - case Some(p) ⇒ httpClientBuilder.setProxy(new HttpHost(p.hostname, p.port)) - case None ⇒ httpClientBuilder - } addProxy(HttpClients.custom(). setConnectionManager(connectionManager(factory(timeout), timeout)). - setDefaultRequestConfig(HTTP.requestConfig(timeout))).build() + setDefaultRequestConfig(HTTP.requestConfig(timeout)), proxy).build() } def readPem(pem: java.io.File)(implicit fileSystem: Effect[FileSystem]) = { val content = fileSystem().readStream(pem)(is ⇒ Source.fromInputStream(is).mkString) From 404fa3aaf8f6dfb771c335a4fb48e93c8edad244 Mon Sep 17 00:00:00 2001 From: Julien Perret Date: Tue, 26 Jun 2018 22:22:54 +0200 Subject: [PATCH 4/4] Toward the completion of the HTTP proxy handling... --- .../src/main/scala/gridscale/dirac/package.scala | 8 ++++---- egi/src/main/scala/gridscale/egi/EGI.scala | 10 +++++----- .../egi/dirac/src/main/scala/TestDIRAC.scala | 6 +++--- .../webdav/src/main/scala/WedDAVExample.scala | 5 +++-- http/src/main/scala/gridscale/http/package.scala | 16 ++++++++-------- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/dirac/src/main/scala/gridscale/dirac/package.scala b/dirac/src/main/scala/gridscale/dirac/package.scala index 077f865d..0186eda8 100644 --- a/dirac/src/main/scala/gridscale/dirac/package.scala +++ b/dirac/src/main/scala/gridscale/dirac/package.scala @@ -260,14 +260,14 @@ package object dirac { object DIRAC { - class Interpreters { + class Interpreters(proxy: Option[HTTPProxy] = None) { implicit val fileSystemInterpreter = FileSystem() implicit val systemInterpreter = System() - implicit val httpInterpreter = HTTP() + implicit val httpInterpreter = HTTP(proxy) } - def apply[T](f: Interpreters ⇒ T) = { - val interpreters = new Interpreters() + def apply[T](f: Interpreters ⇒ T, proxy: Option[HTTPProxy] = None) = { + val interpreters = new Interpreters(proxy) f(interpreters) } diff --git a/egi/src/main/scala/gridscale/egi/EGI.scala b/egi/src/main/scala/gridscale/egi/EGI.scala index 204ced97..8a6742d1 100644 --- a/egi/src/main/scala/gridscale/egi/EGI.scala +++ b/egi/src/main/scala/gridscale/egi/EGI.scala @@ -4,17 +4,17 @@ import effectaside._ import gridscale.http._ object EGI { - class Interpreters { - implicit val http: Effect[HTTP] = HTTP() + class Interpreters(proxy: Option[HTTPProxy] = None) { + implicit val http: Effect[HTTP] = HTTP(proxy) implicit val fileSystem: Effect[FileSystem] = FileSystem() implicit val bdii: Effect[BDII] = BDII() implicit val system: Effect[System] = System() } - def apply() = new Interpreters + def apply(proxy: Option[HTTPProxy] = None) = new Interpreters(proxy) - def apply[T](f: Interpreters ⇒ T) = { - val intp = new Interpreters + def apply[T](f: Interpreters ⇒ T)(proxy: Option[HTTPProxy]) = { + val intp = new Interpreters(proxy) f(intp) } } \ No newline at end of file diff --git a/examples/egi/dirac/src/main/scala/TestDIRAC.scala b/examples/egi/dirac/src/main/scala/TestDIRAC.scala index 3e8d9fd5..44a63553 100644 --- a/examples/egi/dirac/src/main/scala/TestDIRAC.scala +++ b/examples/egi/dirac/src/main/scala/TestDIRAC.scala @@ -7,9 +7,9 @@ import gridscale.http._ object TestDIRAC extends App { - val password = scala.io.Source.fromFile("/home/reuillon/.globus/password").getLines().next().trim - val p12 = P12Authentication(new java.io.File("/home/reuillon/.globus/certificate.p12"), password) - val certificateDirectory = new java.io.File("/home/reuillon/.openmole/simplet/persistent/CACertificates/") + val password = scala.io.Source.fromFile("/home/julien/.globus/password").getLines().next().trim + val p12 = P12Authentication(new java.io.File("/home/julien/.globus/certificate.p12"), password) + val certificateDirectory = new java.io.File("/home/julien/.openmole/DEL1701P003-Ubuntu/persistent/CACertificates/") val description = JobDescription("/bin/uname", "-a", stdOut = Some("output"), outputSandbox = Seq("output")) diff --git a/examples/egi/webdav/src/main/scala/WedDAVExample.scala b/examples/egi/webdav/src/main/scala/WedDAVExample.scala index fe3df6ed..124319c1 100644 --- a/examples/egi/webdav/src/main/scala/WedDAVExample.scala +++ b/examples/egi/webdav/src/main/scala/WedDAVExample.scala @@ -5,6 +5,7 @@ import java.io.ByteArrayInputStream import gridscale.egi._ import gridscale.webdav._ import gridscale.authentication._ +import gridscale.http.HTTPProxy object WedDAVExample extends App { @@ -12,7 +13,7 @@ object WedDAVExample extends App { val p12 = P12Authentication(new java.io.File("/home/reuillon/.globus/certificate.p12"), password) val certificateDirectory = new java.io.File("/home/reuillon/.openmole/simplet/persistent/CACertificates/") val bdiiServer = BDIIServer("topbdii.grif.fr", 2170) - + val proxy: Option[HTTPProxy] = None EGI { impl ⇒ import impl._ @@ -29,5 +30,5 @@ object WedDAVExample extends App { val c = read(webdav, "youpi2.txt") println(c) - } + }(proxy) } \ No newline at end of file diff --git a/http/src/main/scala/gridscale/http/package.scala b/http/src/main/scala/gridscale/http/package.scala index 69273658..0013f72d 100644 --- a/http/src/main/scala/gridscale/http/package.scala +++ b/http/src/main/scala/gridscale/http/package.scala @@ -51,13 +51,13 @@ package object http { } } - def get[T](url: String) = { - implicit val interpreter = HTTP() + def get[T](url: String, proxy: Option[HTTPProxy] = None) = { + implicit val interpreter = HTTP(proxy) read(buildServer(url), "") } - def getStream[T](url: String)(f: InputStream ⇒ T) = { - implicit val interpreter = HTTP() + def getStream[T](url: String, proxy: Option[HTTPProxy] = None)(f: InputStream ⇒ T) = { + implicit val interpreter = HTTP(proxy) readStream[T](buildServer(url), "", f) } @@ -86,9 +86,9 @@ package object http { object HTTP { - def apply() = Effect(new HTTP()) + def apply(proxy: Option[HTTPProxy] = None) = Effect(new HTTP(proxy)) - def client(server: Server, proxy: Option[HTTPProxy] = None) = + def client(server: Server, proxy: Option[HTTPProxy]) = server match { case s: HTTPServer ⇒ httpClient(s.timeout, proxy) case s: HTTPSServer ⇒ HTTPS.newClient(s.socketFactory, s.timeout, proxy) @@ -134,7 +134,7 @@ package object http { } - class HTTP { + class HTTP(proxy: Option[HTTPProxy] = None) { def withInputStream[T](server: Server, path: String, f: (HttpRequest, HttpResponse) ⇒ T, method: HTTPMethod, test: Boolean) = { def fullURI(path: String) = @@ -176,7 +176,7 @@ package object http { def error(e: Throwable) = new HTTP.ConnectionError(s"Error while connecting to ${uri}, method ${method}", e) try { - val httpClient = HTTP.client(server) + val httpClient = HTTP.client(server, proxy) try { val response = httpClient.execute(methodInstance) try {