creating composite builds - krickert/search-api GitHub Wiki

#!/bin/bash
# This script clones production and test repositories and generates a composite build
# configuration using Gradle's Kotlin DSL.
#
# Production repos will be included directly with includeBuild.
# Test repos are included with a dependencySubstitution block so that a dependency declared as
# "com.example.test:<repoName>" will be resolved to the local composite build.
#
# Adjust the repository URLs and assumed group coordinate for test repos 
# --- Define repository arrays ---
production_repos=(
    "[email protected]:yourusername/project1.git"
    "[email protected]:yourusername/project2.git"
    "[email protected]:yourusername/project3.git"
)

test_repos=(
    "[email protected]:yourusername/test_project1.git"
    "[email protected]:yourusername/test_project2.git"
    "[email protected]:yourusername/test_project3.git"
)

# --- Define base directories ---
prod_dir="./projects"
test_dir="./test_projects"

mkdir -p "$prod_dir"
mkdir -p "$test_dir"

# --- Clone Production Repositories ---
for repo in "${production_repos[@]}"; do
    repo_name=$(basename "$repo" .git)
    target_dir="$prod_dir/$repo_name"
    if [ ! -d "$target_dir" ]; then
        echo "Cloning production repo $repo into $target_dir..."
        git clone "$repo" "$target_dir"
    else
        echo "Production repo '$repo_name' already exists at $target_dir. Skipping clone."
    fi
done

# --- Clone Test Repositories ---
for repo in "${test_repos[@]}"; do
    repo_name=$(basename "$repo" .git)
    target_dir="$test_dir/$repo_name"
    if [ ! -d "$target_dir" ]; then
        echo "Cloning test repo $repo into $target_dir..."
        git clone "$repo" "$target_dir"
    else
        echo "Test repo '$repo_name' already exists at $target_dir. Skipping clone."
    fi
done

# --- Generate settings.gradle.kts ---
settings_file="settings.gradle.kts"
echo "// Auto-generated composite build settings" > "$settings_file"

# Add production repositories.
echo "" >> "$settings_file"
echo "val productionRepos = listOf(" >> "$settings_file"
for repo in "${production_repos[@]}"; do
    repo_name=$(basename "$repo" .git)
    echo "    \"$prod_dir/$repo_name\"," >> "$settings_file"
done
echo ")" >> "$settings_file"
echo "productionRepos.forEach { includeBuild(it) }" >> "$settings_file"
echo "" >> "$settings_file"

# Add test repositories with dependency substitution.
# Assuming the test repos publish with group "com.example.test" and artifact name matching the repo name.
for repo in "${test_repos[@]}"; do
    repo_name=$(basename "$repo" .git)
    test_path="$test_dir/$repo_name"
    echo "includeBuild(\"$test_path\") {" >> "$settings_file"
    echo "    dependencySubstitution {" >> "$settings_file"
    echo "        substitute(module(\"com.example.test:$repo_name\")).with(project(\":\"))" >> "$settings_file"
    echo "    }" >> "$settings_file"
    echo "}" >> "$settings_file"
    echo "" >> "$settings_file"
done

echo "Composite build setup complete. Generated $settings_file."

Below is one way to set up your project so that the composite build and all its subprojects (both production and test) are checked into GitLab. In this approach you use Git submodules to β€œattach” each independent repository to your root repository. New developers then simply run one command to clone the entire tree (including submodules), and the Gradle composite build (configured via your Kotlin DSL settings file) pulls everything together.

1. Repository Structure

Organize your root repository like this:

root-project/
β”œβ”€β”€ .gitmodules
β”œβ”€β”€ settings.gradle.kts
β”œβ”€β”€ build.gradle.kts       // (if you need a root build file)
β”œβ”€β”€ projects/              // Production repositories
β”‚   β”œβ”€β”€ project1/          // a submodule
β”‚   β”œβ”€β”€ project2/          // a submodule
β”‚   └── project3/          // a submodule
└── test_projects/         // Test repositories
    β”œβ”€β”€ test_project1/     // a submodule
    β”œβ”€β”€ test_project2/     // a submodule
    └── test_project3/     // a submodule

2. Setting Up Git Submodules

In the root of your repository, create a .gitmodules file that looks like this:

[submodule "projects/project1"]
    path = projects/project1
    url = [email protected]:yourusername/project1.git

[submodule "projects/project2"]
    path = projects/project2
    url = [email protected]:yourusername/project2.git

[submodule "projects/project3"]
    path = projects/project3
    url = [email protected]:yourusername/project3.git

[submodule "test_projects/test_project1"]
    path = test_projects/test_project1
    url = [email protected]:yourusername/test_project1.git

[submodule "test_projects/test_project2"]
    path = test_projects/test_project2
    url = [email protected]:yourusername/test_project2.git

[submodule "test_projects/test_project3"]
    path = test_projects/test_project3
    url = [email protected]:yourusername/test_project3.git

Commit this file along with your folder structure. (When you add a submodule you can also use the command git submodule add <url> <path> to create/update .gitmodules automatically.)

3. The Gradle Composite Build Configuration

In your settings.gradle.kts file at the root, configure the composite build. For example:

// settings.gradle.kts
rootProject.name = "MyCompositeBuild"

// Include production repositories as composite builds.
val productionRepos = listOf(
    "projects/project1",
    "projects/project2",
    "projects/project3"
)
productionRepos.forEach { includeBuild(it) }

// Include test repositories as composite builds with dependency substitution.
// Assume that each test project publishes an artifact with group "com.example.test"
// and an artifact name that matches its directory name.
listOf("test_project1", "test_project2", "test_project3").forEach { testRepoName ->
    includeBuild("test_projects/$testRepoName") {
        dependencySubstitution {
            substitute(module("com.example.test:$testRepoName")).with(project(":"))
        }
    }
}

This configuration ensures that:

  • The production projects are included as composite builds.
  • The test projects are also included and any dependency declared as, for example, 
    testImplementation("com.example.test:test_project1")
    in your main build will be substituted with the corresponding local composite build.

4. One-Liner for New Developers

Once your root repository is checked into GitLab (with the submodules already added and committed), new developers can check out the entire project tree with a single command:

git clone --recursive [email protected]:yourusername/root-project.git

This command clones the root repository and automatically fetches all submodules so that the complete composite build is available locally.

5. Final Notes

  • Submodules vs. Subtrees:
    Using Git submodules keeps the projects independent (each project retains its own Git history and repository) while being connected to the composite build. Make sure your team is comfortable with the typical submodule workflow.

  • IDE Integration:
    When opening the project in IntelliJ IDEA, select the root project. IntelliJ will detect the Gradle composite build (thanks to the settings.gradle.kts file) and import the subprojects accordingly.

This setup allows you to give new developers a single command that retrieves the entire project tree, while maintaining the independence of each repository and allowing you to leverage Gradle’s composite builds with dependency substitution.

6. Creating a multi-project build

import org.gradle.api.services.BuildService
import org.gradle.api.services.BuildServiceParameters
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.TaskAction
import org.gradle.api.DefaultTask
import org.gradle.process.ExecOperations
import javax.inject.Inject

// Project lists
val allProjects = listOf(
    "./projects/project1",
    "./projects/project2",
    "./projects/project3",
    "./projects/project4",
)

val dockerProjects = listOf(
    "./projects/project3",
    "./projects/project4",
)

// Build service for ExecOperations
abstract class ExecService @Inject constructor(
    private val execOps: ExecOperations
) : BuildService<BuildServiceParameters.None> {
    fun runCommand(dir: String, args: List<String>) {
        val isWindows = System.getProperty("os.name").lowercase().contains("windows")
        val gradleCommand = if (isWindows) "./gradlew.bat" else "./gradlew"

        execOps.exec {
            workingDir = File(dir)
            commandLine(listOf(gradleCommand) + args)
        }
    }
}

val execService = gradle.sharedServices.registerIfAbsent("execService", ExecService::class.java) {}

// Reusable task registration
fun registerMultiProjectTask(
    name: String,
    group: String,
    description: String,
    dirs: List<String>,
    commandFor: (String) -> List<String>
) {
    tasks.register(name) {
        this.group = group
        this.description = description

        doLast {
            val service = execService.get()
            dirs.forEach { dir ->
                val args = commandFor(dir)
                println("Running $args in $dir")
                service.runCommand(dir, args)
            }
        }
    }
}

// Task registrations
registerMultiProjectTask(
    name = "allDocker",
    group = "dockerBuild",
    description = "Create all the docker images",
    dirs = dockerProjects,
    commandFor = { listOf("clean", "publishToMavenLocal", "-x", "test") }
)

registerMultiProjectTask(
    name = "cleanAndTestAll",
    group = "verification",
    description = "Clean and run tests for all projects",
    dirs = allProjects,
    commandFor = { listOf("clean", "test") }
)

registerMultiProjectTask(
    name = "buildAndPublishAll",
    group = "build",
    description = "Build all projects without tests and publish to Maven local",
    dirs = allProjects,
    commandFor = { listOf("clean", "build", "publishToMavenLocal", "-x", "test") }
)
⚠️ **GitHub.com Fallback** ⚠️