[FL-3162] Moved ufbt to fbt codebase (#2520)

* scripts: moved ufbt code
* ufbt: fixed tool path
* ufbt: fixed linter/formatter target descriptions
* scripts: ufbt: cleanup
* fbt: moved fap launch target to tools; ufbt fixes
* fbt: fixed missing headers from SDK
* ufbt: removed debug output
* ufbt: moved project template to main codebase
* ufbt: fixed vscode_dist
* ufbt: path naming changes
* fbt: error message for older ufbt versions
* ufbt: docs fixes
* ufbt: fixed build dir location
* fbt: fixes for extapps objcopy
* fbt: extapps: removed extra debug output; fixed formatting
* ufbt: handle launch target for multiple known apps
* ufbt: dropping wrapper; linter fixes
* ufbt: fixed boostrap path
* ufbt: renamed entrypoint
* ufbt: updated vscode config
* ufbt: moved sconsign db location
* ufbt: fixed sconsign path
* fbt: SDK builders rework
* fbt: reworked sdk packaging
* ufbt: additional checks and state processing
* ufbt: fixed sdk state file location
* dist: not packaging pycache
* dump commit json content
* Github: more workflow debug prints
* Github: fix incorrect commit meta extraction in get_env.py
* ufbt, fbt: changed SConsEnvironmentError->StopError
* fbtenv: no longer needs SCRIPT_PATH pre-set
* ufbt: fixed sdk state check
* scripts: exception fixes for storage.py
* scripts: fbtenv: added FBT_TOOLCHAIN_PATH for on Windows for compat
* ufbt: app template: creating .gitkeep for images folder
* ufbt: app template: fixed .gitkeep creation
* docs: formatting fixes for AppManifests; added link to ufbt
* fbt: added link to PyPI for old ufbt versions
* sdk: fixed dir component paths

Co-authored-by: Aleksandr Kutuzov <alleteam@gmail.com>

scripts/sconsdist.py

@@ -1,13 +1,15 @@
 #!/usr/bin/env python3
 
-from flipper.app import App
-from os.path import join, exists, relpath
-from os import makedirs, walk
-from update import Main as UpdateMain
+import json
 import shutil
-import zipfile
 import tarfile
+import zipfile
+from os import makedirs, walk
+from os.path import exists, join, relpath, basename, split
+
 from ansi.color import fg
+from flipper.app import App
+from update import Main as UpdateMain
 
 
 class ProjectDir:
@@ -54,12 +56,19 @@ class Main(App):
         if project_name == "firmware" and filetype != "elf":
             project_name = "full"
 
-        return self.get_dist_file_name(project_name, filetype)
+        dist_target_path = self.get_dist_file_name(project_name, filetype)
+        self.note_dist_component(
+            project_name, filetype, self.get_dist_path(dist_target_path)
+        )
+        return dist_target_path
+
+    def note_dist_component(self, component: str, extension: str, srcpath: str) -> None:
+        self._dist_components[f"{component}.{extension}"] = srcpath
 
     def get_dist_file_name(self, dist_artifact_type: str, filetype: str) -> str:
         return f"{self.DIST_FILE_PREFIX}{self.target}-{dist_artifact_type}-{self.args.suffix}.{filetype}"
 
-    def get_dist_file_path(self, filename: str) -> str:
+    def get_dist_path(self, filename: str) -> str:
         return join(self.output_dir_path, filename)
 
     def copy_single_project(self, project: ProjectDir) -> None:
@@ -69,17 +78,15 @@ class Main(App):
             if exists(src_file := join(obj_directory, f"{project.project}.{filetype}")):
                 shutil.copyfile(
                     src_file,
-                    self.get_dist_file_path(
-                        self.get_project_file_name(project, filetype)
-                    ),
+                    self.get_dist_path(self.get_project_file_name(project, filetype)),
                 )
-        for foldertype in ("sdk", "lib"):
+        for foldertype in ("sdk_headers", "lib"):
             if exists(sdk_folder := join(obj_directory, foldertype)):
-                self.package_zip(foldertype, sdk_folder)
+                self.note_dist_component(foldertype, "dir", sdk_folder)
 
     def package_zip(self, foldertype, sdk_folder):
         with zipfile.ZipFile(
-            self.get_dist_file_path(self.get_dist_file_name(foldertype, "zip")),
+            self.get_dist_path(self.get_dist_file_name(foldertype, "zip")),
             "w",
             zipfile.ZIP_DEFLATED,
         ) as zf:
@@ -94,7 +101,8 @@ class Main(App):
                     )
 
     def copy(self) -> int:
-        self.projects = dict(
+        self._dist_components: dict[str, str] = dict()
+        self.projects: dict[str, ProjectDir] = dict(
             map(
                 lambda pd: (pd.project, pd),
                 map(ProjectDir, self.args.project),
@@ -122,12 +130,18 @@ class Main(App):
             try:
                 shutil.rmtree(self.output_dir_path)
             except Exception as ex:
-                pass
+                self.logger.warn(f"Failed to clean output directory: {ex}")
 
         if not exists(self.output_dir_path):
+            self.logger.debug(f"Creating output directory {self.output_dir_path}")
             makedirs(self.output_dir_path)
 
+        for folder in ("debug", "scripts"):
+            if exists(folder):
+                self.note_dist_component(folder, "dir", folder)
+
         for project in self.projects.values():
+            self.logger.debug(f"Copying {project.project} for {project.target}")
             self.copy_single_project(project)
 
         self.logger.info(
@@ -137,59 +151,133 @@ class Main(App):
         )
 
         if self.args.version:
-            bundle_dir_name = f"{self.target}-update-{self.args.suffix}"[
-                : self.DIST_FOLDER_MAX_NAME_LENGTH
-            ]
-            bundle_dir = join(self.output_dir_path, bundle_dir_name)
-            bundle_args = [
-                "generate",
-                "-d",
-                bundle_dir,
-                "-v",
-                self.args.version,
-                "-t",
-                self.target,
-                "--dfu",
-                self.get_dist_file_path(
-                    self.get_project_file_name(self.projects["firmware"], "dfu")
-                ),
-                "--stage",
-                self.get_dist_file_path(
-                    self.get_project_file_name(self.projects["updater"], "bin")
-                ),
-            ]
-            if self.args.resources:
-                bundle_args.extend(
-                    (
-                        "-r",
-                        self.args.resources,
-                    )
-                )
-            bundle_args.extend(self.other_args)
+            if bundle_result := self.bundle_update_package():
+                return bundle_result
 
-            if (bundle_result := UpdateMain(no_exit=True)(bundle_args)) == 0:
-                self.logger.info(
-                    fg.boldgreen(
-                        f"Use this directory to self-update your Flipper:\n\t{bundle_dir}"
-                    )
-                )
-
-                # Create tgz archive
-                with tarfile.open(
-                    join(
-                        self.output_dir_path,
-                        f"{self.DIST_FILE_PREFIX}{bundle_dir_name}.tgz",
-                    ),
-                    "w:gz",
-                    compresslevel=9,
-                    format=tarfile.USTAR_FORMAT,
-                ) as tar:
-                    tar.add(bundle_dir, arcname=bundle_dir_name)
-
-            return bundle_result
+        required_components = ("firmware.elf", "full.bin", "update.dir")
+        if all(
+            map(
+                lambda c: c in self._dist_components,
+                required_components,
+            )
+        ):
+            self.bundle_sdk()
 
         return 0
 
+    def bundle_sdk(self):
+        self.logger.info("Bundling SDK")
+        components_paths = dict()
+
+        sdk_components_keys = (
+            "full.bin",
+            "firmware.elf",
+            "update.dir",
+            "sdk_headers.dir",
+            "lib.dir",
+            "debug.dir",
+            "scripts.dir",
+        )
+
+        with zipfile.ZipFile(
+            self.get_dist_path(self.get_dist_file_name("sdk", "zip")),
+            "w",
+            zipfile.ZIP_DEFLATED,
+        ) as zf:
+            for component_key in sdk_components_keys:
+                component_path = self._dist_components.get(component_key)
+                components_paths[component_key] = basename(component_path)
+
+                if component_key.endswith(".dir"):
+                    for root, dirnames, files in walk(component_path):
+                        if "__pycache__" in dirnames:
+                            dirnames.remove("__pycache__")
+                        for file in files:
+                            zf.write(
+                                join(root, file),
+                                join(
+                                    components_paths[component_key],
+                                    relpath(
+                                        join(root, file),
+                                        component_path,
+                                    ),
+                                ),
+                            )
+                else:
+                    zf.write(component_path, basename(component_path))
+
+            zf.writestr(
+                "components.json",
+                json.dumps(
+                    {
+                        "meta": {
+                            "hw_target": self.target,
+                            "flavor": self.flavor,
+                            "version": self.args.version,
+                        },
+                        "components": components_paths,
+                    }
+                ),
+            )
+
+    def bundle_update_package(self):
+        self.logger.debug(
+            f"Generating update bundle with version {self.args.version} for {self.target}"
+        )
+        bundle_dir_name = f"{self.target}-update-{self.args.suffix}"[
+            : self.DIST_FOLDER_MAX_NAME_LENGTH
+        ]
+        bundle_dir = self.get_dist_path(bundle_dir_name)
+        bundle_args = [
+            "generate",
+            "-d",
+            bundle_dir,
+            "-v",
+            self.args.version,
+            "-t",
+            self.target,
+            "--dfu",
+            self.get_dist_path(
+                self.get_project_file_name(self.projects["firmware"], "dfu")
+            ),
+            "--stage",
+            self.get_dist_path(
+                self.get_project_file_name(self.projects["updater"], "bin")
+            ),
+        ]
+        if self.args.resources:
+            bundle_args.extend(
+                (
+                    "-r",
+                    self.args.resources,
+                )
+            )
+        bundle_args.extend(self.other_args)
+
+        if (bundle_result := UpdateMain(no_exit=True)(bundle_args)) == 0:
+            self.note_dist_component("update", "dir", bundle_dir)
+            self.logger.info(
+                fg.boldgreen(
+                    f"Use this directory to self-update your Flipper:\n\t{bundle_dir}"
+                )
+            )
+
+            # Create tgz archive
+            with tarfile.open(
+                join(
+                    self.output_dir_path,
+                    bundle_tgz := f"{self.DIST_FILE_PREFIX}{bundle_dir_name}.tgz",
+                ),
+                "w:gz",
+                compresslevel=9,
+                format=tarfile.USTAR_FORMAT,
+            ) as tar:
+                self.note_dist_component(
+                    "update", "tgz", self.get_dist_path(bundle_tgz)
+                )
+                tar.add(bundle_dir, arcname=bundle_dir_name)
+        return bundle_result
+
 
 if __name__ == "__main__":
     Main()()