deploy: incremental static rebuilds

Makefile

@@ -11,4 +11,4 @@ deploy:
 	./deploy/deploy.sh
 
 static:
-	./deploy/static.sh -o "$$(realpath .)"
+	./deploy/static_incremental.sh -o "$$(realpath .)"

config.yaml.example

@@ -3,6 +3,7 @@ cookie_host: .example.org
 admin_email: admin@example.org
 ic_email: idb.kniga@gmail.com
 
+projects: ['ic', 'foreignone', 'omnia']
 default_project: 'foreignone'
 subdomains:
   dev: foreignone

deploy/deploy.sh

@@ -1,18 +1,22 @@
 #!/bin/sh
 
 die() {
-    >&2 echo "error: $@"
+    >&2 echo "error: $*"
     exit 1
 }
 
 set -e
 
+if ! command -v yq >/dev/null 2>&1; then
+    die "yq is not installed. Please install yq to parse YAML files."
+fi
+
 DIR=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
 
 DEV_DIR="$(realpath "$DIR/../")"
 STAGING_DIR="$HOME/staging"
 PROD_DIR="$HOME/www"
-REPO_URI=$(cat "$DEV_DIR/config.yaml" | grep ^git_repo | head -1 | awk '{print $2}')
+REPO_URI=$(yq -r '.git_repo' "$DEV_DIR/config.yaml")
 
 [ "$(hostname)" = "4in1" ] || die "unexpected hostname. are you sure you're running it on the right machine?"
 
@@ -29,6 +33,10 @@ if [ ! -d .git ]; then
 fi
 
 git reset --hard
+
+# Store the current commit hash before pulling
+PREV_COMMIT=$(git rev-parse HEAD)
+
 git pull origin master
 
 composer install --no-dev --optimize-autoloader --ignore-platform-reqs
@@ -39,7 +47,7 @@ if [ ! -d node_modules ]; then
 fi
 
 cp "$DEV_DIR/config.yaml" .
-"$DIR/static.sh" -o "$STAGING_DIR"
+"$DIR/static_incremental.sh" -o "$STAGING_DIR" -p "$PREV_COMMIT"
 
 cd "$DIR"
 

deploy/static.sh

@@ -1,12 +1,16 @@
 #!/bin/sh
 
 die() {
-    >&2 echo "error: $@"
+    >&2 echo "error: $*"
     exit 1
 }
 
 set -e
 
+if ! command -v yq >/dev/null 2>&1; then
+    die "yq is not installed. Please install yq to parse YAML files."
+fi
+
 PHP="$(which php)"
 SCRIPT_DIR=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
 APP_DIR="$(realpath "$SCRIPT_DIR/../")"
@@ -22,11 +26,12 @@ while [ $# -gt 0 ]; do
 done
 [ -z "$OUTPUT_ROOT_DIR" ] && die "you must specify output directory"
 
-for project in ic foreignone omnia; do
+PROJECTS=$("$SCRIPT_DIR"/util/get_projects.sh "$APP_DIR/config.yaml")
-	"$SCRIPT_DIR"/util/build_css.sh -i "$APP_DIR/public/$project/scss" -o "$OUTPUT_ROOT_DIR/public/$project/dist-css"                 || die "build_css failed"
+for project in $PROJECTS; do
-	"$SCRIPT_DIR"/util/build_js.sh -i "$APP_DIR/public/common/js" -o "$OUTPUT_ROOT_DIR/public/$project/dist-js"                       || die "build_js failed"
+	"$SCRIPT_DIR"/util/build_css.sh -i "$APP_DIR/public/$project/scss" -o "$OUTPUT_ROOT_DIR/public/$project/dist-css" || die "build_css failed"
+	"$SCRIPT_DIR"/util/build_js.sh -i "$APP_DIR/public/common/js" -o "$OUTPUT_ROOT_DIR/public/$project/dist-js" || die "build_js failed"
 	$PHP "$SCRIPT_DIR"/util/gen_runtime_config.php \
 		--app-root "$OUTPUT_ROOT_DIR" \
-		--commit-hash "$(git rev-parse --short=8 HEAD)" \
+		--commit-hash "$(git rev-parse HEAD)" \
 		> "$OUTPUT_ROOT_DIR/config-runtime.php" || die "gen_runtime_config failed"
 done

deploy/static_incremental.sh

@@ -0,0 +1,84 @@
+#!/bin/sh
+
+die() {
+    >&2 echo "error: $*"
+    exit 1
+}
+
+set -e
+
+PHP="$(which php)"
+SCRIPT_DIR=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
+APP_DIR="$(realpath "$SCRIPT_DIR/../")"
+OUTPUT_ROOT_DIR=
+PREV_COMMIT=
+
+usage() {
+    cat <<EOF
+usage: $(basename "$0") [OPTIONS]
+
+Options:
+    -o  output directory
+    -p  previous commit hash
+    -h  show this help
+EOF
+    exit 1
+}
+
+while [ $# -gt 0 ]; do
+    case $1 in
+        -o) OUTPUT_ROOT_DIR="$2"; shift ;;
+        -p) PREV_COMMIT="$2"; shift ;;
+        -h) usage ;;
+        *) die "unexpected argument: $1" ;;
+    esac
+    shift
+done
+[ -z "$OUTPUT_ROOT_DIR" ] && die "you must specify output directory"
+
+# Get current commit hash
+CURR_COMMIT=$(git rev-parse HEAD)
+
+# Detect which projects have changed
+CHANGED_PROJECTS=$("$SCRIPT_DIR"/util/detect_changes.sh -a "$APP_DIR" -p "$PREV_COMMIT" -c "$CURR_COMMIT")
+
+# If no projects have changed, just update the commit hash in config-runtime.php
+if [ -z "$CHANGED_PROJECTS" ]; then
+    echo "No static files have changed, only updating commit hash in config-runtime.php"
+    $PHP "$SCRIPT_DIR"/util/gen_runtime_config_incremental.php \
+        --app-root "$OUTPUT_ROOT_DIR" \
+        --commit-hash "$CURR_COMMIT" \
+        --config-file "$OUTPUT_ROOT_DIR/config-runtime.php" \
+        > "$OUTPUT_ROOT_DIR/config-runtime.php.new" || die "gen_runtime_config_incremental failed"
+    mv "$OUTPUT_ROOT_DIR/config-runtime.php.new" "$OUTPUT_ROOT_DIR/config-runtime.php"
+    exit 0
+fi
+
+echo "Rebuilding static for projects: $CHANGED_PROJECTS"
+
+# Create output directories if they don't exist
+for project in $CHANGED_PROJECTS; do
+    mkdir -p "$OUTPUT_ROOT_DIR/public/$project/dist-css"
+    mkdir -p "$OUTPUT_ROOT_DIR/public/$project/dist-js"
+done
+
+# Build static for changed projects
+for project in $CHANGED_PROJECTS; do
+    echo "Building CSS for $project..."
+    "$SCRIPT_DIR"/util/build_css.sh -i "$APP_DIR/public/$project/scss" -o "$OUTPUT_ROOT_DIR/public/$project/dist-css" || die "build_css failed for $project"
+
+    echo "Building JS for $project..."
+    "$SCRIPT_DIR"/util/build_js.sh -i "$APP_DIR/public/common/js" -o "$OUTPUT_ROOT_DIR/public/$project/dist-js" || die "build_js failed for $project"
+done
+
+# Generate runtime config with integrity hashes
+echo "Generating runtime config..."
+$PHP "$SCRIPT_DIR"/util/gen_runtime_config_incremental.php \
+    --app-root "$OUTPUT_ROOT_DIR" \
+    --commit-hash "$CURR_COMMIT" \
+    --changed-projects "$CHANGED_PROJECTS" \
+    --config-file "$OUTPUT_ROOT_DIR/config-runtime.php" \
+    > "$OUTPUT_ROOT_DIR/config-runtime.php.new" || die "gen_runtime_config_incremental failed"
+mv "$OUTPUT_ROOT_DIR/config-runtime.php.new" "$OUTPUT_ROOT_DIR/config-runtime.php"
+
+echo "Static build completed successfully"

deploy/util/build_common.sh

@@ -5,17 +5,18 @@ set -e
 
 INDIR=
 OUTDIR=
+PRESERVE_OUTPUT=0
 
 error() {
-    >&2 echo "error: $@"
+    >&2 echo "error: $*"
 }
 
 warning() {
-    >&2 echo "warning: $@"
+    >&2 echo "warning: $*"
 }
 
 die() {
-    error "$@"
+    error "$*"
     exit 1
 }
 
@@ -28,6 +29,7 @@ usage: $PROGNAME [OPTIONS]
 Options:
     -o  output directory
     -i  input directory
+    -p  preserve output directory (don't clear it)
     -h  show this help
 EOF
     exit "$code"
@@ -39,6 +41,7 @@ input_args() {
         case $1 in
             -o) OUTDIR="$2"; shift ;;
             -i) INDIR="$2"; shift ;;
+            -p) PRESERVE_OUTPUT=1 ;;
             -h) usage ;;
             *) die "unexpected argument: $1" ;;
         esac
@@ -57,7 +60,7 @@ check_args() {
     }
     if [ ! -d "$OUTDIR" ]; then
         mkdir "$OUTDIR"
-    else
+    elif [ "$PRESERVE_OUTPUT" -eq 0 ]; then
         find "$OUTDIR" -mindepth 1 -delete
     fi
-}
+}

deploy/util/detect_changes.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+
+# This script detects changes in static files after git pull
+# It outputs a list of subprojects that need to be rebuilt
+
+set -e
+
+die() {
+    >&2 echo "error: $*"
+    exit 1
+}
+
+if ! command -v yq >/dev/null 2>&1; then
+    die "yq is not installed. Please install yq to parse YAML files."
+fi
+
+APP_DIR=
+PREV_COMMIT=
+CURR_COMMIT=
+
+usage() {
+    cat <<EOF
+usage: $(basename "$0") [OPTIONS]
+
+Options:
+    -a  app directory
+    -p  previous commit hash
+    -c  current commit hash
+    -h  show this help
+EOF
+    exit 1
+}
+
+while [ $# -gt 0 ]; do
+    case $1 in
+        -a) APP_DIR="$2"; shift ;;
+        -p) PREV_COMMIT="$2"; shift ;;
+        -c) CURR_COMMIT="$2"; shift ;;
+        -h) usage ;;
+        *) die "unexpected argument: $1" ;;
+    esac
+    shift
+done
+
+[ -z "$APP_DIR" ] && die "app directory not specified"
+[ -z "$CURR_COMMIT" ] && die "current commit hash not specified"
+
+SCRIPT_DIR=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
+PROJECTS=$("$SCRIPT_DIR"/get_projects.sh "$APP_DIR/config.yaml")
+
+# If no previous commit is specified, assume all projects need to be rebuilt
+if [ -z "$PREV_COMMIT" ]; then
+    echo "$PROJECTS"
+    exit 0
+fi
+
+# Check if common files have changed
+common_changed=$(git diff --name-only "$PREV_COMMIT" "$CURR_COMMIT" -- "$APP_DIR/public/common")
+if [ -n "$common_changed" ]; then
+    # If common files have changed, all projects need to be rebuilt
+    echo "$PROJECTS"
+    exit 0
+fi
+
+# Check which project-specific files have changed
+projects_changed=""
+for project in $PROJECTS; do
+    project_changed=$(git diff --name-only "$PREV_COMMIT" "$CURR_COMMIT" -- "$APP_DIR/public/$project")
+    if [ -n "$project_changed" ]; then
+        projects_changed="$projects_changed $project"
+    fi
+done
+
+# Output the list of projects that need to be rebuilt
+echo "$projects_changed"

deploy/util/gen_runtime_config.php

@@ -2,6 +2,13 @@
 <?php
 
 require __DIR__.'/../../src/init.php';
+global $config;
+
+// Check if yaml extension is available
+if (!function_exists('yaml_parse_file')) {
+    fwrite(STDERR, "error: yaml extension is not installed. Please install php-yaml extension.\n");
+    exit(1);
+}
 
 $commit_hash = null;
 $app_root = null;
@@ -28,7 +35,7 @@ $hashes = [
     'assets' => []
 ];
 
-foreach (['ic', 'foreignone', 'omnia'] as $project) {
+foreach ($config['projects'] as $project) {
     foreach (['js', 'css'] as $type) {
         $dist_dir = $app_root.'/public/'.$project.'/dist-'.$type;
         $entries = glob_recursive($dist_dir.'/*.'.$type);
@@ -81,4 +88,4 @@ function glob_recursive(string $pattern, int $flags = 0): array {
         $files = array_merge($files, glob_recursive($dir.'/'.basename($pattern), $flags));
     }
     return $files;
-}
+}

deploy/util/gen_runtime_config_incremental.php

@@ -0,0 +1,117 @@
+#!/usr/bin/env php
+<?php
+
+require __DIR__.'/../../src/init.php';
+global $config;
+
+// Check if yaml extension is available
+if (!function_exists('yaml_parse_file')) {
+    fwrite(STDERR, "error: yaml extension is not installed. Please install php-yaml extension.\n");
+    exit(1);
+}
+
+$commit_hash = null;
+$app_root = null;
+$changed_projects = [];
+$config_file = null;
+
+for ($i = 1; $i < $argc; $i++) {
+    switch ($argv[$i]) {
+        case '--commit-hash':
+            $commit_hash = $argv[++$i] ?? usage('missing value for --commit-hash');
+            break;
+
+        case '--app-root':
+            $app_root = $argv[++$i] ?? usage('missing value for --app-root');
+            break;
+
+        case '--changed-projects':
+            $changed_projects_str = $argv[++$i] ?? usage('missing value for --changed-projects');
+            $changed_projects = explode(' ', trim($changed_projects_str));
+            break;
+
+        case '--config-file':
+            $config_file = $argv[++$i] ?? usage('missing value for --config-file');
+            break;
+
+        default:
+            usage("unknown option {$argv[$i]}");
+    }
+}
+
+if (is_null($commit_hash) || is_null($app_root))
+    usage();
+
+$all_projects = $config['projects'];
+
+// Load existing config if available
+$hashes = [
+    'commit_hash' => $commit_hash,
+    'assets' => []
+];
+
+if (!empty($config_file) && file_exists($config_file)) {
+    $existing_hashes = include $config_file;
+    if (is_array($existing_hashes) && isset($existing_hashes['assets'])) {
+        $hashes['assets'] = $existing_hashes['assets'];
+    }
+}
+
+// Process only changed projects or all if none specified
+$projects = !empty($changed_projects) ? $changed_projects : $all_projects;
+
+foreach ($projects as $project) {
+    foreach (['js', 'css'] as $type) {
+        $dist_dir = $app_root.'/public/'.$project.'/dist-'.$type;
+        $entries = glob_recursive($dist_dir.'/*.'.$type);
+        if (empty($entries)) {
+            fwrite(STDERR, "warning: no files found in $dist_dir\n");
+            continue;
+        }
+        foreach ($entries as $file) {
+            $asset_key = $type.'/'.basename($file);
+            $hashes['assets'][$project][$asset_key] = [
+                'integrity' => []
+            ];
+            foreach (\engine\skin\FeaturedSkin::RESOURCE_INTEGRITY_HASHES as $hash_type) {
+                $hashes['assets'][$project][$asset_key]['integrity'][$hash_type] = base64_encode(hash_file($hash_type, $file, true));
+            }
+        }
+    }
+}
+
+echo "<?php\n\n";
+echo "return ".var_export($hashes, true).";\n";
+
+function usage(string $msg = ''): never  {
+    if ($msg !== '')
+        fwrite(STDERR, "error: {$msg}\n");
+    $script = $GLOBALS['argv'][0];
+    fwrite(STDERR, "usage: {$script} --commit-hash HASH --app-root APP_ROOT [--changed-projects PROJECTS] [--config-file FILE]\n");
+    exit(1);
+}
+
+function glob_escape(string $pattern): string {
+    if (str_contains($pattern, '[') || str_contains($pattern, ']')) {
+        $placeholder = uniqid();
+        $replaces = [$placeholder.'[', $placeholder.']', ];
+        $pattern = str_replace( ['[', ']'], $replaces, $pattern);
+        $pattern = str_replace( $replaces, ['[[]', '[]]'], $pattern);
+    }
+    return $pattern;
+}
+
+/**
+ * Does not support flag GLOB_BRACE
+ *
+ * @param string $pattern
+ * @param int $flags
+ * @return array
+ */
+function glob_recursive(string $pattern, int $flags = 0): array {
+    $files = glob(glob_escape($pattern), $flags);
+    foreach (glob(glob_escape(dirname($pattern)).'/*', GLOB_ONLYDIR|GLOB_NOSORT) as $dir) {
+        $files = array_merge($files, glob_recursive($dir.'/'.basename($pattern), $flags));
+    }
+    return $files;
+}

deploy/util/get_projects.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+# This script parses the projects list from config.yaml using yq
+# It outputs a space-separated list of projects
+
+die() {
+    >&2 echo "error: $*"
+    exit 1
+}
+
+if ! command -v yq >/dev/null 2>&1; then
+    die "yq is not installed. Please install yq to parse YAML files."
+fi
+
+CONFIG_FILE="$1"
+[ -z "$CONFIG_FILE" ] && die "config file not specified"
+[ -f "$CONFIG_FILE" ] || die "config file not found: $CONFIG_FILE"
+
+# Parse projects list from config.yaml
+PROJECTS=$(yq -r '.projects | join(" ")' "$CONFIG_FILE")
+
+# Check if projects list is empty
+[ -z "$PROJECTS" ] && die "projects list is empty in $CONFIG_FILE"
+
+echo "$PROJECTS"