Skip to content
This repository was archived by the owner on Aug 17, 2025. It is now read-only.

Attempt: engine redo #5694

Draft
matt2e wants to merge 11 commits into
base: main
Choose a base branch
from
Draft

Attempt: engine redo #5694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e328aef
attempt to make one watcher
matt2e Jun 3, 2025
3dd7962
fix jvm build not being returned to engine
matt2e Jun 4, 2025
6bd98b7
fix file watching
matt2e Jun 4, 2025
cfc4198
remove stack trace
matt2e Jun 4, 2025
2aaeac3
another attempt
matt2e Jun 5, 2025
28efce1
progress
matt2e Jun 6, 2025
7a5ba37
clean up and bug fixes
matt2e Jun 6, 2025
72ae902
remove old engine
matt2e Jun 6, 2025
8291ff8
fix idle + stopping after build
matt2e Jun 11, 2025
baf1368
fix deploy
matt2e Jun 11, 2025
b617c87
remove module meta
matt2e Jun 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 4 additions & 31 deletions backend/protos/xyz/block/ftl/buildengine/v1/buildengine.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 0 additions & 3 deletions backend/protos/xyz/block/ftl/buildengine/v1/buildengine.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,20 +40,17 @@ message ModuleBuildWaiting {
// ModuleBuildStarted is published when a build has started for a module.
message ModuleBuildStarted {
xyz.block.ftl.language.v1.ModuleConfig config = 1;
bool is_auto_rebuild = 2;
}

// ModuleBuildFailed is published for any build failures.
message ModuleBuildFailed {
xyz.block.ftl.language.v1.ModuleConfig config = 1;
xyz.block.ftl.language.v1.ErrorList errors = 2;
bool is_auto_rebuild = 3;
}

// ModuleBuildSuccess is published when all modules have been built successfully built.
message ModuleBuildSuccess {
xyz.block.ftl.language.v1.ModuleConfig config = 1;
bool is_auto_rebuild = 2;
}

// ModuleDeployStarted is published when a deploy has been queued
Expand Down
14 changes: 6 additions & 8 deletions cmd/ftl/cmd_build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
errors "github.com/alecthomas/errors"

"github.com/block/ftl/backend/protos/xyz/block/ftl/admin/v1/adminpbconnect"
"github.com/block/ftl/common/log"
"github.com/block/ftl/internal/buildengine"
"github.com/block/ftl/internal/projectconfig"
"github.com/block/ftl/internal/schema/schemaeventsource"
Expand All @@ -24,7 +23,7 @@ func (b *buildCmd) Run(
schemaSource *schemaeventsource.EventSource,
projConfig projectconfig.Config,
) error {
logger := log.FromContext(ctx)
// logger := log.FromContext(ctx)
if len(b.Dirs) == 0 {
b.Dirs = projConfig.AbsModuleDirs()
}
Expand All @@ -38,7 +37,6 @@ func (b *buildCmd) Run(
engine, err := buildengine.New(
ctx,
adminClient,
schemaSource,
projConfig,
b.Dirs,
false,
Expand All @@ -48,11 +46,11 @@ func (b *buildCmd) Run(
if err != nil {
return errors.WithStack(err)
}
if len(engine.Modules()) == 0 {
logger.Warnf("No modules were found to build")
return nil
}
if err := engine.Build(ctx); err != nil {
// if len(engine.Modules()) == 0 {
// logger.Warnf("No modules were found to build")
// return nil
// }
if err := engine.Build(ctx, schemaSource); err != nil {
return errors.Wrap(err, "build failed")
}
return nil
Expand Down
183 changes: 88 additions & 95 deletions cmd/ftl/cmd_buildimage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,8 @@ package main

import (
"context"
"os"
"path/filepath"

errors "github.com/alecthomas/errors"

"github.com/block/ftl"
"github.com/block/ftl/backend/protos/xyz/block/ftl/admin/v1/adminpbconnect"
"github.com/block/ftl/common/key"
"github.com/block/ftl/common/log"
"github.com/block/ftl/common/schema"
"github.com/block/ftl/internal/buildengine"
"github.com/block/ftl/internal/oci"
"github.com/block/ftl/internal/projectconfig"
"github.com/block/ftl/internal/schema/schemaeventsource"
Expand All @@ -36,94 +27,96 @@ func (b *buildImageCmd) Run(
schemaSource *schemaeventsource.EventSource,
projConfig projectconfig.Config,
) error {
logger := log.FromContext(ctx)
if len(b.Dirs) == 0 {
b.Dirs = projConfig.AbsModuleDirs()
}
if len(b.Dirs) == 0 {
return errors.WithStack(errors.New("no directories specified"))
}
// TODO: redo all this

// logger := log.FromContext(ctx)
// if len(b.Dirs) == 0 {
// b.Dirs = projConfig.AbsModuleDirs()
// }
// if len(b.Dirs) == 0 {
// return errors.WithStack(errors.New("no directories specified"))
// }

// Cancel build engine context to ensure all language plugins are killed.
ctx, cancel := context.WithCancelCause(ctx)
defer cancel(errors.Wrap(context.Canceled, "build stopped"))
engine, err := buildengine.New(
ctx,
adminClient,
schemaSource,
projConfig,
b.Dirs,
false,
buildengine.BuildEnv(b.BuildEnv),
buildengine.Parallelism(b.Parallelism),
)
if err != nil {
return errors.WithStack(err)
}
if len(engine.Modules()) == 0 {
logger.Warnf("No modules were found to build")
return nil
}
artefactService, err := oci.NewArtefactService(ctx, b.ArtefactConfig)
if err != nil {
return errors.Wrapf(err, "failed to init artefact service")
}
imageService, err := oci.NewImageService(ctx, artefactService, &b.ImageConfig)
if err != nil {
return errors.Wrapf(err, "failed to init OCI")
}
if err := engine.BuildWithCallback(ctx, func(ctx context.Context, module buildengine.Module, moduleSch *schema.Module, tmpDeployDir string, deployPaths []string) error {
artifacts := []*schema.MetadataArtefact{}
// // Cancel build engine context to ensure all language plugins are killed.
// ctx, cancel := context.WithCancelCause(ctx)
// defer cancel(errors.Wrap(context.Canceled, "build stopped"))
// engine, err := buildengine.New(
// ctx,
// adminClient,
// schemaSource,
// projConfig,
// b.Dirs,
// false,
// buildengine.BuildEnv(b.BuildEnv),
// buildengine.Parallelism(b.Parallelism),
// )
// if err != nil {
// return errors.WithStack(err)
// }
// if len(engine.Modules()) == 0 {
// logger.Warnf("No modules were found to build")
// return nil
// }
// artefactService, err := oci.NewArtefactService(ctx, b.ArtefactConfig)
// if err != nil {
// return errors.Wrapf(err, "failed to init artefact service")
// }
// imageService, err := oci.NewImageService(ctx, artefactService, &b.ImageConfig)
// if err != nil {
// return errors.Wrapf(err, "failed to init OCI")
// }
// if err := engine.BuildWithCallback(ctx, func(ctx context.Context, module buildengine.Module, moduleSch *schema.Module, tmpDeployDir string, deployPaths []string) error {
// artifacts := []*schema.MetadataArtefact{}

for _, i := range deployPaths {
s, err := os.Stat(i)
if err != nil {
return errors.Wrapf(err, "failed to stat file")
}
// for _, i := range deployPaths {
// s, err := os.Stat(i)
// if err != nil {
// return errors.Wrapf(err, "failed to stat file")
// }

path, err := filepath.Rel(tmpDeployDir, i)
if err != nil {
return errors.Wrapf(err, "failed to resolve file")
}
executable := s.Mode().Perm()&0111 != 0
artifacts = append(artifacts, &schema.MetadataArtefact{Path: path, Executable: executable})
}
var image string
if b.RunnerImage != "" {
image = b.RunnerImage
} else {
image = "ftl0/ftl-runner"
if moduleSch.ModRuntime().Base.Image != "" {
image = moduleSch.ModRuntime().Base.Image
}
image += ":"
if ftl.IsRelease(ftl.Version) && ftl.Version == ftl.BaseVersion(ftl.Version) {
image += "v"
image += ftl.Version
} else {
image += "latest"
}
}
tgt := string(b.ArtefactConfig.Repository)
tgt += ":"
tgt += b.Tag
targets := []oci.ImageTarget{}
if !b.SkipLocalDaemon {
targets = append(targets, oci.WithLocalDeamon())
}
if b.Push {
targets = append(targets, oci.WithRemotePush())
}
// TODO: we need to properly sync the deployment with the actual deployment key
// this is just a hack to get the module and realm to the runner
deployment := key.NewDeploymentKey(projConfig.Name, moduleSch.Name)
err := imageService.BuildOCIImage(ctx, image, tgt, tmpDeployDir, deployment, artifacts, targets...)
if err != nil {
return errors.Wrapf(err, "failed to build image")
}
return nil
}); err != nil {
return errors.Wrap(err, "build failed")
}
// path, err := filepath.Rel(tmpDeployDir, i)
// if err != nil {
// return errors.Wrapf(err, "failed to resolve file")
// }
// executable := s.Mode().Perm()&0111 != 0
// artifacts = append(artifacts, &schema.MetadataArtefact{Path: path, Executable: executable})
// }
// var image string
// if b.RunnerImage != "" {
// image = b.RunnerImage
// } else {
// image = "ftl0/ftl-runner"
// if moduleSch.ModRuntime().Base.Image != "" {
// image = moduleSch.ModRuntime().Base.Image
// }
// image += ":"
// if ftl.IsRelease(ftl.Version) && ftl.Version == ftl.BaseVersion(ftl.Version) {
// image += "v"
// image += ftl.Version
// } else {
// image += "latest"
// }
// }
// tgt := string(b.ArtefactConfig.Repository)
// tgt += ":"
// tgt += b.Tag
// targets := []oci.ImageTarget{}
// if !b.SkipLocalDaemon {
// targets = append(targets, oci.WithLocalDeamon())
// }
// if b.Push {
// targets = append(targets, oci.WithRemotePush())
// }
// // TODO: we need to properly sync the deployment with the actual deployment key
// // this is just a hack to get the module and realm to the runner
// deployment := key.NewDeploymentKey(projConfig.Name, moduleSch.Name)
// err := imageService.BuildOCIImage(ctx, image, tgt, tmpDeployDir, deployment, artifacts, targets...)
// if err != nil {
// return errors.Wrapf(err, "failed to build image")
// }
// return nil
// }); err != nil {
// return errors.Wrap(err, "build failed")
// }
return nil
}
Loading