feat: Introducing deb and rpm auto-updates (#7060)

introducing RPM and Deb auto-updates by adding a self-identifying `package-type` resource file to each deb/rpm package

Author: mmaietta

Diff for: .changeset/seven-garlics-tell.md

@@ -0,0 +1,6 @@
+---
+"app-builder-lib": minor
+"electron-updater": minor
+---
+
+feat: Introducing deb and rpm auto-updates as beta feature

Diff for: .github/workflows/test.yaml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       matrix:
         testFiles:
-          - ArtifactPublisherTest,BuildTest,ExtraBuildTest,RepoSlugTest,binDownloadTest,configurationValidationTest,filenameUtilTest,filesTest,globTest,ignoreTest,macroExpanderTest,mainEntryTest,urlUtilTest,extraMetadataTest,linuxArchiveTest,linuxPackagerTest,HoistedNodeModuleTest,PublishManagerTest
+          - ArtifactPublisherTest,BuildTest,ExtraBuildTest,RepoSlugTest,binDownloadTest,configurationValidationTest,filenameUtilTest,filesTest,globTest,ignoreTest,macroExpanderTest,mainEntryTest,urlUtilTest,extraMetadataTest,linuxArchiveTest,linuxPackagerTest,HoistedNodeModuleTest
           - snapTest,debTest,fpmTest,protonTest
     steps:
       - name: Checkout code repository
@@ -69,7 +69,7 @@ jobs:
       - name: Test
         run: pnpm ci:test
         env:
-          TEST_FILES: masTest,dmgTest,protonTest
+          TEST_FILES: masTest,dmgTest,protonTest,filesTest
           FORCE_COLOR: 1
 
   # Need to separate from other tests because logic is specific to when TOKEN env vars are set
@@ -88,7 +88,7 @@ jobs:
       - name: Test
         run: pnpm ci:test
         env:
-          TEST_FILES: nsisUpdaterTest
+          TEST_FILES: nsisUpdaterTest,linuxUpdaterTest,PublishManagerTest
           KEYGEN_TOKEN: ${{ secrets.KEYGEN_TOKEN }}
           BITBUCKET_TOKEN: ${{ secrets.BITBUCKET_TOKEN }}
           GH_TOKEN: ${{ secrets.GH_TOKEN }}

Diff for: docs/api/electron-builder.md

@@ -1430,6 +1430,7 @@ return path.join(target.outDir, <code>__${target.name}-${getArtifactArchName(arc
 <li><a href="#module_electron-updater.AppUpdater+quitAndInstall"><code>.quitAndInstall(isSilent, isForceRunAfter)</code></a></li>
 </ul>
 </li>
+<li><a href="#DebUpdater">.DebUpdater</a> ⇐ <code>module:electron-updater/out/BaseUpdater.BaseUpdater</code></li>
 <li><a href="#MacUpdater">.MacUpdater</a> ⇐ <code><a href="#AppUpdater">AppUpdater</a></code>
 <ul>
 <li><a href="#module_electron-updater.MacUpdater+quitAndInstall"><code>.quitAndInstall()</code></a></li>
@@ -1450,6 +1451,7 @@ return path.join(target.outDir, <code>__${target.name}-${getArtifactArchName(arc
 <li><a href="#module_electron-updater.Provider+resolveFiles"><code>.resolveFiles(updateInfo)</code></a> ⇒ <code>Array&lt;<a href="#ResolvedUpdateFileInfo">ResolvedUpdateFileInfo</a>&gt;</code></li>
 </ul>
 </li>
+<li><a href="#RpmUpdater">.RpmUpdater</a> ⇐ <code>module:electron-updater/out/BaseUpdater.BaseUpdater</code></li>
 <li><a href="#UpdaterSignal">.UpdaterSignal</a>
 <ul>
 <li><a href="#module_electron-updater.UpdaterSignal+login"><code>.login(handler)</code></a></li>
@@ -1752,7 +1754,11 @@ This is different from the normal quit event sequence.</p>
 </tr>
 </tbody>
 </table>
-<p><a name="MacUpdater"></a></p>
+<p><a name="DebUpdater"></a></p>
+<h2 id="debupdater-%E2%87%90-module%3Aelectron-updater%2Fout%2Fbaseupdater.baseupdater">DebUpdater ⇐ <code>module:electron-updater/out/BaseUpdater.BaseUpdater</code></h2>
+<p><strong>Kind</strong>: class of <a href="#module_electron-updater"><code>electron-updater</code></a><br/>
+<strong>Extends</strong>: <code>module:electron-updater/out/BaseUpdater.BaseUpdater</code><br>
+<a name="MacUpdater"></a></p>
 <h2 id="macupdater-%E2%87%90-appupdater">MacUpdater ⇐ <code><a href="#AppUpdater">AppUpdater</a></code></h2>
 <p><strong>Kind</strong>: class of <a href="#module_electron-updater"><code>electron-updater</code></a><br/>
 <strong>Extends</strong>: <code><a href="#AppUpdater">AppUpdater</a></code></p>
@@ -1910,7 +1916,11 @@ This is different from the normal quit event sequence.</p>
 </tr>
 </tbody>
 </table>
-<p><a name="UpdaterSignal"></a></p>
+<p><a name="RpmUpdater"></a></p>
+<h2 id="rpmupdater-%E2%87%90-module%3Aelectron-updater%2Fout%2Fbaseupdater.baseupdater">RpmUpdater ⇐ <code>module:electron-updater/out/BaseUpdater.BaseUpdater</code></h2>
+<p><strong>Kind</strong>: class of <a href="#module_electron-updater"><code>electron-updater</code></a><br/>
+<strong>Extends</strong>: <code>module:electron-updater/out/BaseUpdater.BaseUpdater</code><br>
+<a name="UpdaterSignal"></a></p>
 <h2 id="updatersignal">UpdaterSignal</h2>
 <p><strong>Kind</strong>: class of <a href="#module_electron-updater"><code>electron-updater</code></a><br/></p>
 <ul>

Diff for: docs/configuration/publish.md

@@ -133,7 +133,7 @@ In all publish options <a href="/file-patterns#file-macros">File Macros</a> are
 <p><code id="GenericServerOptions-requestHeaders">requestHeaders</code> module:http.OutgoingHttpHeaders - Any custom request headers</p>
 </li>
 <li>
-<p><code id="GenericServerOptions-timeout">timeout</code> = <code>60000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
+<p><code id="GenericServerOptions-timeout">timeout</code> = <code>120000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
 </li>
 </ul>
 <h2 id="githuboptions">GithubOptions</h2>
@@ -183,7 +183,7 @@ Define <code>GH_TOKEN</code> environment variable.</p>
 <p><code id="GithubOptions-requestHeaders">requestHeaders</code> module:http.OutgoingHttpHeaders - Any custom request headers</p>
 </li>
 <li>
-<p><code id="GithubOptions-timeout">timeout</code> = <code>60000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
+<p><code id="GithubOptions-timeout">timeout</code> = <code>120000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
 </li>
 </ul>
 <h2 id="snapstoreoptions">SnapStoreOptions</h2>
@@ -203,7 +203,7 @@ Define <code>GH_TOKEN</code> environment variable.</p>
 <p><code id="SnapStoreOptions-requestHeaders">requestHeaders</code> module:http.OutgoingHttpHeaders - Any custom request headers</p>
 </li>
 <li>
-<p><code id="SnapStoreOptions-timeout">timeout</code> = <code>60000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
+<p><code id="SnapStoreOptions-timeout">timeout</code> = <code>120000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
 </li>
 </ul>
 <h2 id="spacesoptions">SpacesOptions</h2>
@@ -238,7 +238,7 @@ Define <code>KEYGEN_TOKEN</code> environment variable.</p>
 <p><code id="KeygenOptions-requestHeaders">requestHeaders</code> module:http.OutgoingHttpHeaders - Any custom request headers</p>
 </li>
 <li>
-<p><code id="KeygenOptions-timeout">timeout</code> = <code>60000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
+<p><code id="KeygenOptions-timeout">timeout</code> = <code>120000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
 </li>
 </ul>
 <h2 id="bitbucketoptions">BitbucketOptions</h2>
@@ -269,7 +269,7 @@ Define <code>BITBUCKET_TOKEN</code> environment variable.</p>
 <p><code id="BitbucketOptions-requestHeaders">requestHeaders</code> module:http.OutgoingHttpHeaders - Any custom request headers</p>
 </li>
 <li>
-<p><code id="BitbucketOptions-timeout">timeout</code> = <code>60000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
+<p><code id="BitbucketOptions-timeout">timeout</code> = <code>120000</code> Number | “undefined” - Request timeout in milliseconds. (Default is 2 minutes; O is ignored)</p>
 </li>
 </ul>
 <h2 id="s3options">S3Options</h2>

Diff for: package.json

@@ -20,7 +20,7 @@
     "///": "Please see https://github.com/electron-userland/electron-builder/blob/master/CONTRIBUTING.md#run-test-using-cli how to run particular test instead full (and very slow) run",
     "test": "node ./test/out/helpers/runTests.js skipArtifactPublisher",
     "test-all": "pnpm compile && pnpm pretest && pnpm ci:test",
-    "test-linux": "docker run --rm -e UPDATE_SNAPSHOT=${UPDATE_SNAPSHOT:-false} -e TEST_FILES=\"${TEST_FILES:-HoistedNodeModuleTest}\" -v $(pwd):/project -v $(pwd)-node-modules:/project/node_modules -v $HOME/Library/Caches/electron:/root/.cache/electron -v $HOME/Library/Caches/electron-builder:/root/.cache/electron-builder electronuserland/builder:wine-mono /bin/bash -c \"pnpm install && node ./test/out/helpers/runTests.js\"",
+    "test-linux": "docker run --rm -e DEBUG=${DEBUG:-} -e UPDATE_SNAPSHOT=${UPDATE_SNAPSHOT:-false} -e TEST_FILES=\"${TEST_FILES:-HoistedNodeModuleTest}\" -v $(pwd):/project -v $(pwd)-node-modules:/project/node_modules -v $HOME/Library/Caches/electron:/root/.cache/electron -v $HOME/Library/Caches/electron-builder:/root/.cache/electron-builder electronuserland/builder:wine-mono /bin/bash -c \"pnpm install && node ./test/out/helpers/runTests.js\"",
     "test-update": "UPDATE_SNAPSHOT=true pnpm test-all",
     "docker-images": "docker/build.sh",
     "docker-push": "docker/push.sh",

Diff for: packages/app-builder-lib/src/linuxPackager.ts

@@ -6,7 +6,7 @@ import { PlatformPackager } from "./platformPackager"
 import { RemoteBuilder } from "./remoteBuilder/RemoteBuilder"
 import AppImageTarget from "./targets/AppImageTarget"
 import FlatpakTarget from "./targets/FlatpakTarget"
-import FpmTarget from "./targets/fpm"
+import FpmTarget from "./targets/FpmTarget"
 import { LinuxTargetHelper } from "./targets/LinuxTargetHelper"
 import SnapTarget from "./targets/snap"
 import { createCommonTarget } from "./targets/targetFactory"
@@ -57,7 +57,7 @@ export class LinuxPackager extends PlatformPackager<LinuxConfiguration> {
           case "pacman":
           case "apk":
           case "p5p":
-            return require("./targets/fpm").default
+            return require("./targets/FpmTarget").default
           default:
             return null
         }

Diff for: packages/app-builder-lib/src/publish/PublishManager.ts

@@ -109,9 +109,6 @@ export class PublishManager implements PublishContext {
         if (!event.targets.some(it => isSuitableWindowsTarget(it))) {
           return
         }
-      } else {
-        // AppImage writes data to AppImage stage dir, not to linux-unpacked
-        return
       }
 
       const publishConfig = await getAppUpdatePublishConfiguration(packager, event.arch, this.isPublish)

Diff for: packages/app-builder-lib/src/targets/fpm.ts renamed to packages/app-builder-lib/src/targets/FpmTarget.ts

@@ -1,7 +1,7 @@
 import { path7za } from "7zip-bin"
-import { Arch, executeAppBuilder, log, TmpDir, toLinuxArchString, use } from "builder-util"
+import { Arch, executeAppBuilder, getArchSuffix, log, TmpDir, toLinuxArchString, use, serializeToYaml } from "builder-util"
 import { unlinkIfExists } from "builder-util/out/fs"
-import { outputFile } from "fs-extra"
+import { outputFile, stat } from "fs-extra"
 import { mkdir, readFile } from "fs/promises"
 import * as path from "path"
 import { smarten } from "../appInfo"
@@ -15,6 +15,9 @@ import { isMacOsSierra } from "../util/macosVersion"
 import { getTemplatePath } from "../util/pathManager"
 import { installPrefix, LinuxTargetHelper } from "./LinuxTargetHelper"
 import { getLinuxToolsPath } from "./tools"
+import { hashFile } from "../util/hash"
+import { ArtifactCreated } from "../packagerApi"
+import { getAppUpdatePublishConfiguration } from "../publish/PublishManager"
 
 interface FpmOptions {
   name: string
@@ -109,7 +112,8 @@ export default class FpmTarget extends Target {
     }
 
     const packager = this.packager
-    const artifactPath = path.join(this.outDir, packager.expandArtifactNamePattern(this.options, target, arch, nameFormat, !isUseArchIfX64))
+    const artifactName = packager.expandArtifactNamePattern(this.options, target, arch, nameFormat, !isUseArchIfX64)
+    const artifactPath = path.join(this.outDir, artifactName)
 
     await packager.info.callArtifactBuildStarted({
       targetPresentableName: target,
@@ -122,6 +126,18 @@ export default class FpmTarget extends Target {
       await mkdir(this.outDir, { recursive: true })
     }
 
+    const publishConfig = this.supportsAutoUpdate(target)
+      ? await getAppUpdatePublishConfiguration(packager, arch, false /* in any case validation will be done on publish */)
+      : null
+    if (publishConfig != null) {
+      const linuxDistType = this.packager.packagerOptions.prepackaged || path.join(this.outDir, `linux${getArchSuffix(arch)}-unpacked`)
+      const resourceDir = packager.getResourcesDir(linuxDistType)
+      log.info({ resourceDir }, `adding autoupdate files for: ${target}. (Beta feature)`)
+      await outputFile(path.join(resourceDir, "app-update.yml"), serializeToYaml(publishConfig))
+      // Extra file needed for auto-updater to detect installation method
+      await outputFile(path.join(resourceDir, "package-type"), target)
+    }
+
     const scripts = await this.scriptFiles
     const appInfo = packager.appInfo
     const options = this.options
@@ -229,7 +245,28 @@ export default class FpmTarget extends Target {
 
     await executeAppBuilder(["fpm", "--configuration", JSON.stringify(fpmConfiguration)], undefined, { env })
 
-    await packager.dispatchArtifactCreated(artifactPath, this, arch)
+    let info: ArtifactCreated = {
+      file: artifactPath,
+      target: this,
+      arch,
+      packager,
+    }
+    if (publishConfig != null) {
+      info = {
+        ...info,
+        safeArtifactName: packager.computeSafeArtifactName(artifactName, target, arch, !isUseArchIfX64),
+        isWriteUpdateInfo: true,
+        updateInfo: {
+          sha512: await hashFile(artifactPath),
+          size: (await stat(artifactPath)).size,
+        },
+      }
+    }
+    await packager.info.callArtifactBuildCompleted(info)
+  }
+
+  private supportsAutoUpdate(target: string) {
+    return ["deb", "rpm"].includes(target)
   }
 }
 

Diff for: packages/builder-util-runtime/src/publishOptions.ts

@@ -50,7 +50,7 @@ export interface PublishConfiguration {
   /**
    * Request timeout in milliseconds. (Default is 2 minutes; O is ignored)
    *
-   * @default 60000
+   * @default 120000
    */
   readonly timeout?: number | null
 }

Diff for: packages/electron-updater/src/AppImageUpdater.ts

@@ -1,5 +1,5 @@
 import { AllPublishOptions, newError } from "builder-util-runtime"
-import { execFileSync, spawn } from "child_process"
+import { execFileSync } from "child_process"
 import { chmod } from "fs-extra"
 import { unlinkSync } from "fs"
 import * as path from "path"
@@ -30,7 +30,7 @@ export class AppImageUpdater extends BaseUpdater {
   /*** @private */
   protected doDownloadUpdate(downloadUpdateOptions: DownloadUpdateOptions): Promise<Array<string>> {
     const provider = downloadUpdateOptions.updateInfoAndProvider.provider
-    const fileInfo = findFile(provider.resolveFiles(downloadUpdateOptions.updateInfoAndProvider.info), "AppImage")!
+    const fileInfo = findFile(provider.resolveFiles(downloadUpdateOptions.updateInfoAndProvider.info), "AppImage", ["rpm", "deb"])!
     return this.executeDownload({
       fileExtension: "AppImage",
       fileInfo,
@@ -99,19 +99,15 @@ export class AppImageUpdater extends BaseUpdater {
     }
 
     const env: any = {
-      ...process.env,
       APPIMAGE_SILENT_INSTALL: "true",
     }
 
     if (options.isForceRunAfter) {
-      spawn(destination, [], {
-        detached: true,
-        stdio: "ignore",
-        env,
-      }).unref()
+      // eslint-disable-next-line @typescript-eslint/no-floating-promises
+      this.spawnLog(destination, [], env)
     } else {
       env.APPIMAGE_EXIT_AFTER_INSTALL = "true"
-      execFileSync(destination, [], { env })
+      execFileSync(destination, [], env)
     }
     return true
   }

Diff for: packages/electron-updater/src/BaseUpdater.ts

@@ -1,4 +1,5 @@
 import { AllPublishOptions } from "builder-util-runtime"
+import { spawn, spawnSync } from "child_process"
 import { AppAdapter } from "./AppAdapter"
 import { AppUpdater, DownloadExecutorTask } from "./AppUpdater"
 
@@ -98,6 +99,60 @@ export abstract class BaseUpdater extends AppUpdater {
       this.install(true, false)
     })
   }
+
+  protected wrapSudo() {
+    const { name } = this.app
+    const installComment = `"${name} would like to update"`
+    const sudo = this.spawnSyncLog("which gksudo || which kdesudo || which pkexec || which beesu")
+    const command = [sudo]
+    if (/kdesudo/i.test(sudo)) {
+      command.push("--comment", installComment)
+      command.push("-c")
+    } else if (/gksudo/i.test(sudo)) {
+      command.push("--message", installComment)
+    } else if (/pkexec/i.test(sudo)) {
+      command.push("--disable-internal-agent")
+    }
+    return command.join(" ")
+  }
+
+  protected spawnSyncLog(cmd: string, args: string[] = [], env = {}): string {
+    this._logger.info(`Executing: ${cmd} with args: ${args}`)
+    const response = spawnSync(cmd, args, {
+      stdio: "pipe",
+      env: { ...process.env, ...env },
+      encoding: "utf-8",
+      shell: true,
+    })
+    return response.stdout.trim()
+  }
+
+  /**
+   * This handles both node 8 and node 10 way of emitting error when spawning a process
+   *   - node 8: Throws the error
+   *   - node 10: Emit the error(Need to listen with on)
+   */
+  // https://github.com/electron-userland/electron-builder/issues/1129
+  // Node 8 sends errors: https://nodejs.org/dist/latest-v8.x/docs/api/errors.html#errors_common_system_errors
+  protected async spawnLog(cmd: string, args: string[] = [], env: any = {}): Promise<boolean> {
+    this._logger.info(`Executing: ${cmd} with args: ${args}`)
+    return new Promise<boolean>((resolve, reject) => {
+      try {
+        const p = spawn(cmd, args, {
+          stdio: "pipe",
+          env: { ...process.env, ...env },
+          detached: true,
+        })
+        p.on("error", error => {
+          reject(error)
+        })
+        p.unref()
+        resolve(p.pid !== undefined)
+      } catch (error) {
+        reject(error)
+      }
+    })
+  }
 }
 
 export interface InstallOptions {

Diff for: packages/electron-updater/src/DebUpdater.ts

@@ -0,0 +1,38 @@
+import { AllPublishOptions } from "builder-util-runtime"
+import { AppAdapter } from "./AppAdapter"
+import { DownloadUpdateOptions } from "./AppUpdater"
+import { BaseUpdater, InstallOptions } from "./BaseUpdater"
+import { DOWNLOAD_PROGRESS } from "./main"
+import { findFile } from "./providers/Provider"
+
+export class DebUpdater extends BaseUpdater {
+  constructor(options?: AllPublishOptions | null, app?: AppAdapter) {
+    super(options, app)
+  }
+
+  /*** @private */
+  protected doDownloadUpdate(downloadUpdateOptions: DownloadUpdateOptions): Promise<Array<string>> {
+    const provider = downloadUpdateOptions.updateInfoAndProvider.provider
+    const fileInfo = findFile(provider.resolveFiles(downloadUpdateOptions.updateInfoAndProvider.info), "deb", ["AppImage", "rpm"])!
+    return this.executeDownload({
+      fileExtension: "deb",
+      fileInfo,
+      downloadUpdateOptions,
+      task: async (updateFile, downloadOptions) => {
+        if (this.listenerCount(DOWNLOAD_PROGRESS) > 0) {
+          downloadOptions.onProgress = it => this.emit(DOWNLOAD_PROGRESS, it)
+        }
+        await this.httpExecutor.download(fileInfo.url, updateFile, downloadOptions)
+      },
+    })
+  }
+
+  protected doInstall(options: InstallOptions): boolean {
+    const sudo = this.wrapSudo()
+    // pkexec doesn't want the command to be wrapped in " quotes
+    const wrapper = /pkexec/i.test(sudo) ? "" : `"`
+    const cmd = ["dpkg", "-i", options.installerPath, "||", "apt-get", "install", "-f", "-y"]
+    this.spawnSyncLog(sudo, [`${wrapper}/bin/bash`, "-c", `'${cmd.join(" ")}'${wrapper}`])
+    return true
+  }
+}