Add support for development environment

.github/workflows/ci.yml

@@ -43,6 +43,10 @@ jobs:
         with:
           DEVELOPMENT_MODE: true
 
+      - name: Setup submodules
+        run: |
+          git submodule init
+          git submodule update
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 

.github/workflows/daily.yml

@@ -34,6 +34,11 @@ jobs:
         with:
           DEVELOPMENT_MODE: true
 
+      - name: Setup submodules
+        run: |
+          git submodule init
+          git submodule update
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "examples/lilac/minimal/mfe_learning_build/frontend-app-learning"]
+	path = examples/lilac/minimal/mfe_learning_build/frontend-app-learning
+	url = https://github.com/openedx/frontend-app-learning.git
+	branch = open-release/lilac.master

README.md

@@ -18,11 +18,20 @@ Derex Plugin to integrate Open edX Learning Microfrontend
 
 There are some options that can be passed to the plugin configuration in your derex.config.yaml file.
 
-- build_dir: An optional build directory which should contain a customized .env.derex file and a Caddyfile which will be included in the build context
-- dockerfile_path: Path to a Dockerfile relative to the derex.config.yaml file location which will be used inplace of the default one
+- docker_image: Then tag which will be given to the built docker image
+- build_dir: An optional build directory which content will be included in the build context. Some files will you'll probably want to include here:
+
+  - `.env.derex.j2` and `.env.development.derex.j2` files
+  - a `Caddyfile`
+  - a `Dockerfile.j2` Jinja template which will compiled and used for the build
+  - the whole microfrontend repository. This is especially useful when doing local development
+  - any additinal file you might need in your build. `.j2` files will be compiled with the derex `Project` object in the context
+
+  If `.env.derex.j2`, `.env.development.derex.j2`, `Caddyfile` and `Dockerfile.j2` are not present default one will be used.
+
 - aliases: Additional network aliases for the docker container. This list will also be used to populate the `CORS_ORIGIN_WHITELIST` and `LOGIN_REDIRECT_WHITELIST` LMS settings
 - NODE_VERSION: The node version which will be given as a build argument
-- MFE_REPOSITORY: A repository URL which will be given as a build argument
+- MFE_REPOSITORY: A repository URL which will be given as a build argument or a path to a local repository in the `build_dir`
 - MFE_BRANCH: A Git branch which will be checked out after cloning the Microfrontend repository
 
 e.g.:
@@ -32,7 +41,7 @@ plugins:
   derex.mfe-learning:
     {
       "build_dir": "mfe_learning_build",
-      "dockerfile_path": "mfe_learning_build/Dockerfile",
+      "docker_image": "my-custom-image-name",
       "aliases": [
         "learning.mydomain.com",
       ]

derex/mfe_learning/cli.py

@@ -1,15 +1,20 @@
-from pathlib import Path
 from typing import Optional
 
 import click
 from derex.runner.build import build_microfrontend_image
 from derex.runner.cli import ensure_project
 from derex.runner.cli.build import build as derex_build_cli
 from derex.runner.project import Project
-from derex.runner.utils import abspath_from_egg
 
 from derex.mfe_learning import __version__
-from derex.mfe_learning.constants import MfeLearningVersions
+from derex.mfe_learning.constants import (
+    DEFAULT_BUILD_DIR,
+    DEFAULT_CADDYFILE_PATH,
+    DEFAULT_DEV_ENV_FILE_TEMPLATE_PATH,
+    DEFAULT_DOCKERFILE_TEMPLATE_PATH,
+    DEFAULT_ENV_FILE_TEMPLATE_PATH,
+    MfeLearningVersions,
+)
 
 
 @derex_build_cli.command("mfe-learning")
@@ -24,8 +29,8 @@
 @click.option(
     "-T",
     "--target",
-    type=str,
-    default="final",
+    type=click.Choice(["sourceonly", "dev", "final"]),
+    required=False,
     help="Target to build",
 )
 @click.option(
@@ -75,6 +80,10 @@ def mfe_learning_build(
 
     if not version:
         version = project.openedx_version
+    if not target:
+        target = "final"
+        if project.runmode.name == "debug":
+            target = "dev"
 
     default_config = MfeLearningVersions[version.name]
 
@@ -84,24 +93,50 @@ def mfe_learning_build(
         click.echo(tag)
         return 0
 
-    default_build_dir = abspath_from_egg(
-        "derex.mfe_learning", "derex/mfe_learning/docker_build/Dockerfile"
-    ).parent
-    build_dir = Path(config.get("build_dir") or default_build_dir)
-    dockerfile_path = Path(
-        config.get("dockerfile_path") or default_build_dir / "Dockerfile"
-    )
+    build_dir = DEFAULT_BUILD_DIR
+    caddyfile_path = DEFAULT_CADDYFILE_PATH
+    dockerfile_template_path = DEFAULT_DOCKERFILE_TEMPLATE_PATH
+    env_file_template_path = DEFAULT_ENV_FILE_TEMPLATE_PATH
+    dev_env_file_template_path = DEFAULT_DEV_ENV_FILE_TEMPLATE_PATH
+
+    if config.get("build_dir"):
+        build_dir = project.root / config.get("build_dir")
 
     if not build_dir.exists() or not build_dir.is_dir():
         raise click.ClickException(
             f"Build dir {build_dir} does not exist or is not a directory. Aborting."
         )
-    if not dockerfile_path.exists() or not dockerfile_path.is_file():
-        raise click.ClickException(
-            f"Dockerfile {dockerfile_path} does not exist or is not a file. Aborting."
-        )
+
+    if build_dir != DEFAULT_BUILD_DIR:
+        if (build_dir / DEFAULT_CADDYFILE_PATH.name).exists() and (
+            build_dir / DEFAULT_CADDYFILE_PATH.name
+        ).is_file():
+            caddyfile_path = build_dir / DEFAULT_CADDYFILE_PATH.name
+        if (build_dir / DEFAULT_DOCKERFILE_TEMPLATE_PATH.name).exists() and (
+            build_dir / DEFAULT_DOCKERFILE_TEMPLATE_PATH.name
+        ).is_file():
+            dockerfile_template_path = build_dir / DEFAULT_DOCKERFILE_TEMPLATE_PATH.name
+        if (build_dir / DEFAULT_ENV_FILE_TEMPLATE_PATH.name).exists() and (
+            build_dir / DEFAULT_ENV_FILE_TEMPLATE_PATH.name
+        ).is_file():
+            env_file_template_path = build_dir / DEFAULT_ENV_FILE_TEMPLATE_PATH.name
+        if (build_dir / DEFAULT_DEV_ENV_FILE_TEMPLATE_PATH.name).exists() and (
+            build_dir / DEFAULT_DEV_ENV_FILE_TEMPLATE_PATH.name
+        ).is_file():
+            dev_env_file_template_path = (
+                build_dir / DEFAULT_DEV_ENV_FILE_TEMPLATE_PATH.name
+            )
 
     paths_to_copy = [path for path in build_dir.iterdir()]
+    for path in [
+        caddyfile_path,
+        dockerfile_template_path,
+        env_file_template_path,
+        dev_env_file_template_path,
+    ]:
+        if path not in paths_to_copy:
+            paths_to_copy.append(path)
+
     build_args = {
         "NODE_VERSION": config.get(
             "NODE_VERSION", default_config.value["NODE_VERSION"]
@@ -125,5 +160,5 @@ def mfe_learning_build(
         cache_from,
         cache_to,
         build_args=build_args,
-        dockerfile_path=dockerfile_path,
+        dockerfile_template_path=dockerfile_template_path,
     )

derex/mfe_learning/config.py

@@ -19,7 +19,7 @@ def generate_local_docker_compose(project: Project) -> Path:
         pkg_resources.resource_filename(__name__, "docker-compose-mfe-learning.yml.j2")
     )
     try:
-        config = project.config.get("plugins").get("mfe-learning") or {}
+        config = project.config.get("plugins").get("derex.mfe-learning") or {}
     except AttributeError:
         config = {}
 
@@ -30,11 +30,20 @@ def generate_local_docker_compose(project: Project) -> Path:
         "docker_image", f"{default_docker_image_prefix}:{__version__}"
     )
     mfe_learning_aliases = config.get("aliases") or []
+    mfe_learning_repository = config.get("MFE_REPOSITORY") or None
+    mfe_learning_build_dir = config.get("build_dir") or None
+
+    if mfe_learning_repository and mfe_learning_build_dir:
+        mfe_learning_repository = Path(mfe_learning_build_dir) / mfe_learning_repository
+        if not mfe_learning_repository.exists() or not mfe_learning_repository.is_dir():
+            mfe_learning_repository = None
+
     tmpl = Template(template_path.read_text())
     text = tmpl.render(
         project=project,
         mfe_learning_docker_image=mfe_learning_docker_image,
         mfe_learning_aliases=mfe_learning_aliases,
+        mfe_learning_repository=mfe_learning_repository,
     )
     local_compose_path.write_text(text)
     return local_compose_path

derex/mfe_learning/constants.py

@@ -1,5 +1,23 @@
 from enum import Enum
 
+from derex.runner.utils import abspath_from_egg
+
+DEFAULT_BUILD_DIR = abspath_from_egg(
+    "derex.mfe_learning", "derex/mfe_learning/docker_build/Dockerfile.j2"
+).parent
+DEFAULT_CADDYFILE_PATH = abspath_from_egg(
+    "derex.mfe_learning", "derex/mfe_learning/docker_build/Caddyfile"
+)
+DEFAULT_DOCKERFILE_TEMPLATE_PATH = abspath_from_egg(
+    "derex.mfe_learning", "derex/mfe_learning/docker_build/Dockerfile.j2"
+)
+DEFAULT_ENV_FILE_TEMPLATE_PATH = abspath_from_egg(
+    "derex.mfe_learning", "derex/mfe_learning/docker_build/.env.derex.j2"
+)
+DEFAULT_DEV_ENV_FILE_TEMPLATE_PATH = abspath_from_egg(
+    "derex.mfe_learning", "derex/mfe_learning/docker_build/.env.development.derex.j2"
+)
+
 
 class MfeLearningVersions(Enum):
     # Values will be passed as uppercased named arguments to the docker build

derex/mfe_learning/docker-compose-mfe-learning.yml.j2

@@ -4,11 +4,19 @@ services:
   mfe-learning:
     container_name: "{{ project.name}}_mfe_learning"
     image: {{ mfe_learning_docker_image }}
+    {% if project.runmode.name == "debug" -%}
+    command: fedx-scripts webpack-dev-server --progress --disable-host-check
+    {% if mfe_learning_repository.is_dir() -%}
+    volumes:
+      - {{ mfe_learning_repository.absolute() }}:/openedx/microfrontend
+    {% endif -%}
+    {% endif -%}
     networks:
       derex:
         aliases:
           - learning.{{ project.name}}.localhost
-          {% for alias in mfe_learning_aliases %}
+          - learning.{{ project.name}}.localhost.derex
+          {% for alias in mfe_learning_aliases -%}
           - {{ alias }}
           {% endfor %}
 

derex/mfe_learning/docker_build/.env.derex.j2

@@ -17,7 +17,7 @@ LOGO_WHITE_URL=https://edx-cdn.org/v3/default/logo-white.svg
 FAVICON_URL=https://edx-cdn.org/v3/default/favicon.ico
 MARKETING_SITE_BASE_URL='http://{{ project.name }}.localhost'
 ORDER_HISTORY_URL='http://localhost:1996/orders'
-PORT=2000
+PORT=80
 REFRESH_ACCESS_TOKEN_ENDPOINT='http://{{ project.name }}.localhost/login_refresh'
 SEARCH_CATALOG_URL='http://{{ project.name }}.localhost/courses'
 SEGMENT_KEY=''

derex/mfe_learning/docker_build/.env.development.derex.j2

@@ -0,0 +1,35 @@
+NODE_ENV='development'
+ACCESS_TOKEN_COOKIE_NAME='edx-jwt-cookie-header-payload'
+BASE_URL='http://learning.{{ project.name }}.localhost'
+CREDENTIALS_BASE_URL=
+CSRF_TOKEN_API_PATH='/csrf/api/v1/token'
+DISCOVERY_API_BASE_URL=
+ECOMMERCE_BASE_URL=
+ENTERPRISE_LEARNER_PORTAL_HOSTNAME=
+IGNORED_ERROR_REGEX=''
+LANGUAGE_PREFERENCE_COOKIE_NAME='openedx-language-preference'
+LMS_BASE_URL='http://{{ project.name }}.localhost'
+LOGIN_URL='http://{{ project.name }}.localhost/login'
+LOGOUT_URL='http://{{ project.name }}.localhost/logout'
+LOGO_URL=https://edx-cdn.org/v3/default/logo.svg
+LOGO_TRADEMARK_URL=https://edx-cdn.org/v3/default/logo-trademark.svg
+LOGO_WHITE_URL=https://edx-cdn.org/v3/default/logo-white.svg
+FAVICON_URL=https://edx-cdn.org/v3/default/favicon.ico
+MARKETING_SITE_BASE_URL='http://{{ project.name }}.localhost'
+ORDER_HISTORY_URL='http://localhost:1996/orders'
+PORT=80
+REFRESH_ACCESS_TOKEN_ENDPOINT='http://{{ project.name }}.localhost/login_refresh'
+SEARCH_CATALOG_URL='http://{{ project.name }}.localhost/courses'
+SEGMENT_KEY=''
+SITE_NAME='edX'
+SOCIAL_UTM_MILESTONE_CAMPAIGN='edxmilestone'
+STUDIO_BASE_URL='http://studio.{{ project.name }}.localhost'
+SUPPORT_URL='https://support.edx.org'
+SUPPORT_URL_CALCULATOR_MATH='https://support.edx.org/hc/en-us/articles/360000038428-Entering-math-expressions-in-assignments-or-the-calculator'
+SUPPORT_URL_ID_VERIFICATION='https://support.edx.org/hc/en-us/articles/206503858-How-do-I-verify-my-identity'
+SUPPORT_URL_VERIFIED_CERTIFICATE='https://support.edx.org/hc/en-us/articles/206502008-What-is-a-verified-certificate'
+TERMS_OF_SERVICE_URL='https://www.edx.org/edx-terms-service'
+TWITTER_HASHTAG='myedxjourney'
+TWITTER_URL='https://twitter.com/edXOnline'
+USER_INFO_COOKIE_NAME='edx-user-info'
+SESSION_COOKIE_DOMAIN='localhost'

examples/lilac/minimal/mfe_learning_build/Dockerfile → derex/mfe_learning/docker_build/Dockerfile.j2

@@ -16,18 +16,28 @@ ENV PATH ./node_modules/.bin:${PATH}
 FROM base AS sourceonly
 ARG MFE_REPOSITORY
 ARG MFE_BRANCH
+
+{% if mfe_repository.is_dir() -%}
 RUN git clone ${MFE_REPOSITORY} \
     --branch ${MFE_BRANCH} --depth 1 /openedx/microfrontend
-RUN npm install
+{% else -%}
+COPY ${MFE_REPOSITORY} /openedx/microfrontend
+{% endif -%}
+
+RUN yarn install --modules-folder /openedx/microfrontend/node_modules
 COPY .env.derex .env
-RUN npm run build
+COPY .env.development.derex .env.development
+
+FROM sourceonly AS dev
+CMD fedx-scripts webpack-dev-server --progress --disable-host-check
+
+FROM sourceonly AS build
+RUN yarn build
 
 FROM docker.io/caddy:2.3.0-alpine AS final
 
 RUN mkdir -p /openedx/dist
-COPY --from=sourceonly \
+COPY --from=build \
     /openedx/microfrontend/dist /srv/microfrontend
 
-COPY test_file.txt /test_file.txt
-
 COPY ./Caddyfile /etc/caddy/Caddyfile

examples/lilac/minimal/derex.config.yaml

@@ -6,4 +6,8 @@ plugins:
     {
       "build_dir": "mfe_learning_build",
       "dockerfile_path": "mfe_learning_build/Dockerfile",
+      "dev_dockerfile_path": "mfe_learning_build/Dockerfile-dev",
+      "MFE_REPOSITORY": "frontend-app-learning",
+      "MFE_BRANCH": "open-release/lilac.master",
+      "NODE_VERSION": "16.13.1-alpine3.13",
     }

examples/lilac/minimal/mfe_learning_build/.env.derex.j2

@@ -17,7 +17,7 @@ LOGO_WHITE_URL=https://abstract-technology.de/++theme++abstract-ng/static/img/lo
 FAVICON_URL=https://edx-cdn.org/v3/default/favicon.ico
 MARKETING_SITE_BASE_URL='http://{{ project.name }}.localhost'
 ORDER_HISTORY_URL='http://localhost:1996/orders'
-PORT=2000
+PORT=80
 REFRESH_ACCESS_TOKEN_ENDPOINT='http://{{ project.name }}.localhost/login_refresh'
 SEARCH_CATALOG_URL='http://{{ project.name }}.localhost/courses'
 SEGMENT_KEY=''

examples/lilac/minimal/mfe_learning_build/.env.development.derex.j2

@@ -0,0 +1,35 @@
+NODE_ENV='development'
+ACCESS_TOKEN_COOKIE_NAME='edx-jwt-cookie-header-payload'
+BASE_URL='http://learning.{{ project.name }}.localhost'
+CREDENTIALS_BASE_URL=
+CSRF_TOKEN_API_PATH='/csrf/api/v1/token'
+DISCOVERY_API_BASE_URL=
+ECOMMERCE_BASE_URL=
+ENTERPRISE_LEARNER_PORTAL_HOSTNAME=
+IGNORED_ERROR_REGEX=''
+LANGUAGE_PREFERENCE_COOKIE_NAME='openedx-language-preference'
+LMS_BASE_URL='http://{{ project.name }}.localhost'
+LOGIN_URL='http://{{ project.name }}.localhost/login'
+LOGOUT_URL='http://{{ project.name }}.localhost/logout'
+LOGO_URL=https://abstract-technology.de/++theme++abstract-ng/static/img/logo_bgblack.png
+LOGO_TRADEMARK_URL=https://abstract-technology.de/++theme++abstract-ng/static/img/logo_bgblack.png
+LOGO_WHITE_URL=https://abstract-technology.de/++theme++abstract-ng/static/img/logo_bgblack.png
+FAVICON_URL=https://edx-cdn.org/v3/default/favicon.ico
+MARKETING_SITE_BASE_URL='http://{{ project.name }}.localhost'
+ORDER_HISTORY_URL='http://localhost:1996/orders'
+PORT=80
+REFRESH_ACCESS_TOKEN_ENDPOINT='http://{{ project.name }}.localhost/login_refresh'
+SEARCH_CATALOG_URL='http://{{ project.name }}.localhost/courses'
+SEGMENT_KEY=''
+SITE_NAME='edX'
+SOCIAL_UTM_MILESTONE_CAMPAIGN='edxmilestone'
+STUDIO_BASE_URL='http://studio.{{ project.name }}.localhost'
+SUPPORT_URL='https://support.edx.org'
+SUPPORT_URL_CALCULATOR_MATH='https://support.edx.org/hc/en-us/articles/360000038428-Entering-math-expressions-in-assignments-or-the-calculator'
+SUPPORT_URL_ID_VERIFICATION='https://support.edx.org/hc/en-us/articles/206503858-How-do-I-verify-my-identity'
+SUPPORT_URL_VERIFIED_CERTIFICATE='https://support.edx.org/hc/en-us/articles/206502008-What-is-a-verified-certificate'
+TERMS_OF_SERVICE_URL='https://www.edx.org/edx-terms-service'
+TWITTER_HASHTAG='myedxjourney'
+TWITTER_URL='https://twitter.com/edXOnline'
+USER_INFO_COOKIE_NAME='edx-user-info'
+SESSION_COOKIE_DOMAIN='localhost'