commit c0f5a07c80d383bf87d304e484a2bac635812ce6 Author: Savanni D'Gerinel Date: Sun Jul 9 17:06:25 2023 -0400 Set up the documentation for the mononix project diff --git a/Mononix/.obsidian/app.json b/Mononix/.obsidian/app.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/Mononix/.obsidian/app.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/Mononix/.obsidian/appearance.json b/Mononix/.obsidian/appearance.json new file mode 100644 index 0000000..c8c365d --- /dev/null +++ b/Mononix/.obsidian/appearance.json @@ -0,0 +1,3 @@ +{ + "accentColor": "" +} \ No newline at end of file diff --git a/Mononix/.obsidian/backlink.json b/Mononix/.obsidian/backlink.json new file mode 100644 index 0000000..48ee913 --- /dev/null +++ b/Mononix/.obsidian/backlink.json @@ -0,0 +1,3 @@ +{ + "backlinkInDocument": true +} \ No newline at end of file diff --git a/Mononix/.obsidian/core-plugins-migration.json b/Mononix/.obsidian/core-plugins-migration.json new file mode 100644 index 0000000..5c13490 --- /dev/null +++ b/Mononix/.obsidian/core-plugins-migration.json @@ -0,0 +1,29 @@ +{ + "file-explorer": true, + "global-search": true, + "switcher": true, + "graph": true, + "backlink": true, + "canvas": true, + "outgoing-link": true, + "tag-pane": true, + "page-preview": true, + "daily-notes": true, + "templates": true, + "note-composer": true, + "command-palette": true, + "slash-command": false, + "editor-status": true, + "bookmarks": true, + "markdown-importer": false, + "zk-prefixer": false, + "random-note": false, + "outline": true, + "word-count": true, + "slides": false, + "audio-recorder": false, + "workspaces": false, + "file-recovery": true, + "publish": false, + "sync": false +} \ No newline at end of file diff --git a/Mononix/.obsidian/core-plugins.json b/Mononix/.obsidian/core-plugins.json new file mode 100644 index 0000000..9405bfd --- /dev/null +++ b/Mononix/.obsidian/core-plugins.json @@ -0,0 +1,20 @@ +[ + "file-explorer", + "global-search", + "switcher", + "graph", + "backlink", + "canvas", + "outgoing-link", + "tag-pane", + "page-preview", + "daily-notes", + "templates", + "note-composer", + "command-palette", + "editor-status", + "bookmarks", + "outline", + "word-count", + "file-recovery" +] \ No newline at end of file diff --git a/Mononix/.obsidian/hotkeys.json b/Mononix/.obsidian/hotkeys.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/Mononix/.obsidian/hotkeys.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/Mononix/.obsidian/workspace.json b/Mononix/.obsidian/workspace.json new file mode 100644 index 0000000..f56b5ea --- /dev/null +++ b/Mononix/.obsidian/workspace.json @@ -0,0 +1,184 @@ +{ + "main": { + "id": "009f4879d7d58688", + "type": "split", + "children": [ + { + "id": "8b2d8fed8c2820e7", + "type": "tabs", + "children": [ + { + "id": "0168bf593828da5a", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "cargo2nix.md", + "mode": "source", + "backlinks": true, + "source": false + } + } + }, + { + "id": "f8131571839b6055", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "Index.md", + "mode": "source", + "backlinks": true, + "source": false + } + } + } + ], + "currentTab": 1 + } + ], + "direction": "vertical" + }, + "left": { + "id": "5e13cf68a0856016", + "type": "split", + "children": [ + { + "id": "3efb5df25c9dd56d", + "type": "tabs", + "children": [ + { + "id": "7e1974ad240e7bbf", + "type": "leaf", + "state": { + "type": "file-explorer", + "state": { + "sortOrder": "alphabetical" + } + } + }, + { + "id": "cdeb4c8f33227fff", + "type": "leaf", + "state": { + "type": "search", + "state": { + "query": "", + "matchingCase": false, + "explainSearch": false, + "collapseAll": false, + "extraContext": false, + "sortOrder": "alphabetical" + } + } + }, + { + "id": "5523bc2dc5f0334c", + "type": "leaf", + "state": { + "type": "bookmarks", + "state": {} + } + } + ] + } + ], + "direction": "horizontal", + "width": 300, + "collapsed": true + }, + "right": { + "id": "00a3036558f626b2", + "type": "split", + "children": [ + { + "id": "cfbf00e95752a62c", + "type": "tabs", + "children": [ + { + "id": "fcd5a94438929e0e", + "type": "leaf", + "state": { + "type": "backlink", + "state": { + "file": "Index.md", + "collapseAll": false, + "extraContext": false, + "sortOrder": "alphabetical", + "showSearch": false, + "searchQuery": "", + "backlinkCollapsed": false, + "unlinkedCollapsed": true + } + } + }, + { + "id": "0d9ae37b7a736590", + "type": "leaf", + "state": { + "type": "outgoing-link", + "state": { + "file": "Index.md", + "linksCollapsed": false, + "unlinkedCollapsed": true + } + } + }, + { + "id": "9dffaafe6dfb655f", + "type": "leaf", + "state": { + "type": "tag", + "state": { + "sortOrder": "frequency", + "useHierarchy": true + } + } + }, + { + "id": "5ca8ea0c5c458175", + "type": "leaf", + "state": { + "type": "outline", + "state": { + "file": "Index.md" + } + } + } + ] + } + ], + "direction": "horizontal", + "width": 300, + "collapsed": true + }, + "left-ribbon": { + "hiddenItems": { + "switcher:Open quick switcher": false, + "graph:Open graph view": false, + "canvas:Create new canvas": false, + "daily-notes:Open today's daily note": false, + "templates:Insert template": false, + "command-palette:Open command palette": false + } + }, + "active": "f8131571839b6055", + "lastOpenFiles": [ + "Nix For Development.md", + "Nix should immediately make a person's time better.md", + "Nix Flakes.md", + "buildRustPackage.md", + "buildRustCrate.md", + "crate2nix.md", + "cargo2nix.md", + "How can I interrogate a Nix builder.md", + "Index.md", + "Nix Monorepos.md", + "Cross-compiling.md", + "Makefiles and Monorepos.md", + "Mononix, C library.md", + "Mononix, Rust library.md", + "Scaling Rust Builds with Bazel.md", + "Crane.md" + ] +} \ No newline at end of file diff --git a/Mononix/Crane.md b/Mononix/Crane.md new file mode 100644 index 0000000..c61f4f7 --- /dev/null +++ b/Mononix/Crane.md @@ -0,0 +1,5 @@ +- Dependency cache strategy: one derivation for all dependencies. +- Cross-compilation: +- Suitable for Nixpkgs: + +Lack of per-crate caching makes this unsuitable for monorepos. \ No newline at end of file diff --git a/Mononix/Cross-compiling.md b/Mononix/Cross-compiling.md new file mode 100644 index 0000000..0c90cb0 --- /dev/null +++ b/Mononix/Cross-compiling.md @@ -0,0 +1,30 @@ +# 2023-07-09 + +I did some basic things to cross-compile to raspberry pi, and I got an executable that would theoretically run on a Pi. However, the Pi is a Nixos system, so I'll need to either create a full enclosure of my dev environment, or I will need to patchelf anything that I copy over. Patchelf is not impossible, but it's not as easy as just copying the executable from one system to another. + +rust-toolchain: +``` +[toolchain] +channel = "1.68.2" +targets = [ "wasm32-unknown-unknown", "arm-unknown-linux-gnueabihf" ] +``` + +.cargo/config.toml: +``` +[target.arm-unknown-linux-gnueabihf] +linker = "armv6l-unknown-linux-gnueabihf-gcc" +``` + +flake.nix: +``` + armPkgs = import nixpkgs { + system = "x86_64-linux"; + crossSystem = pkgs.lib.systems.examples.raspberryPi; + }; + + ... + + buildInputs = [ + armPkgs.stdenv.cc + ] +``` diff --git a/Mononix/How can I interrogate a Nix builder.md b/Mononix/How can I interrogate a Nix builder.md new file mode 100644 index 0000000..d661d45 --- /dev/null +++ b/Mononix/How can I interrogate a Nix builder.md @@ -0,0 +1,5 @@ +How can I interrogate a Nix builder? + +Nix derivations are often horrifyingly difficult to read. It is also difficult to figure out how to wrangle Nix into giving up any information about how it's interpreting the derivation. + +In order to understand these tools that I'm working with, I need to be able to inject the equivalent of debug prints and have a very clear place to get the results back. \ No newline at end of file diff --git a/Mononix/Index.md b/Mononix/Index.md new file mode 100644 index 0000000..2131b54 --- /dev/null +++ b/Mononix/Index.md @@ -0,0 +1,54 @@ +Tools for running builds using Nix in a monorepo. + +Goal: Using one coherent build tool, I can develop multiple projects with common dependencies. + +Requirements: +- select the toolchains and versions thereof for the project +- configure cross-compilation +- allow multiple languages +- co-exist with language ecosystems. For example, rust-analyzer needs cargo in order to understand Cargo.toml. However, it should be easier to fully build the project with Nix +- every module is compiled into a derivation that downstream modules can import. Each module derivation is cached separately so that Nix only rebuilds the changed module. +- Nix can generate a closure for an executable, even a cross-compiled one, to copy to the remote system + +# Languages and Bindings + +Languages: +- C +- Rust +- Typescript +- Javascript + +Bindings: +- C -> Rust +- Rust -> Typescript +- Rust -> Javascript + +Backends + +| | Linux x86 | Linux ARM | WASM | AVR | ESP32 | +| ---------- | --------- | --------- | ---- | --- | ----- | +| C | yes | yes | | yes | | +| Rust | yes | yes | yes | yes | yes | +| Typescript | | | yes | | | +| Javascript | | | yes | | | + +- [[Mononix, C library]] +- [[Mononix, Rust library]] + +# Build targets + +- build +- run +- tests +- code coverage profiling +- linting +- performance testing + +Each of these is a separate Nix command that builds or runs the desired result. + +--- +- [[Nix Monorepos]] +- [[Nix For Development]] +- [[Cross-compiling]] +- [[Scaling Rust Builds with Bazel]] +- [[Makefiles and Monorepos]] \ No newline at end of file diff --git a/Mononix/Makefiles and Monorepos.md b/Mononix/Makefiles and Monorepos.md new file mode 100644 index 0000000..7ae303f --- /dev/null +++ b/Mononix/Makefiles and Monorepos.md @@ -0,0 +1,6 @@ +Old, tried and true. Make doesn't care at all about dependency management, so I'm going to try a bunch of Makefiles to at least kick off language-native toolchains. I may need more sophisticated rules to make make library/application dependencies work well within the repository. + +Drawbacks: + +- I don't know how to create what amounts to a standard makefile with regular entry points +- There's still a lot of glue that I have to work out for each language. \ No newline at end of file diff --git a/Mononix/Mononix, C library.md b/Mononix/Mononix, C library.md new file mode 100644 index 0000000..f3ca9ce --- /dev/null +++ b/Mononix/Mononix, C library.md @@ -0,0 +1,6 @@ + +Exports: +- Header files +- Native DLL +- Native static library +- Dependencies diff --git a/Mononix/Mononix, Rust library.md b/Mononix/Mononix, Rust library.md new file mode 100644 index 0000000..6c7eaeb --- /dev/null +++ b/Mononix/Mononix, Rust library.md @@ -0,0 +1,12 @@ + +Exports: +- Rust libaries +- Native DLL +- Native static library +- Dependencies + +A Rust library can have both native system dependencies (in the form of other libraries) and Rust dependencies (in the form of Rust crates). + +A Rust library can, with effort, be published with C bindings. + +[[Are all Rust DLLs and static libraries meant for import to C?]] diff --git a/Mononix/Nix Flakes.md b/Mononix/Nix Flakes.md new file mode 100644 index 0000000..1784ff5 --- /dev/null +++ b/Mononix/Nix Flakes.md @@ -0,0 +1,39 @@ +3 part tutorial by Eelco Dolstra + +- solves the problem where build inputs are dependent on environment variables, the nix channels, and so forth. These are inconsistent across systems. +- *no standard way to compose nix projects* +- packages can provide their own `flake.nix` file + - `nix shell github:edolstra/dwarffs --command dwarffs --version` +- flakes must declare dependencies, and dependencies must be locked to specific versions +- `nix flake metadata` will show the flake's dependencies +- `nix flake show` will show all of the outputs +- `defaultPackage.` must be a derivation +- flakes are specified with a URL-like syntax: `github:edolstra/dwarffs` or `git+https://github.com/NixOS/patchelf`. The flake registery maps symbolic identifiers to actual locations. `nixpkgs` is equivalent to `github:NixOS/nixpkgs`. +- Registry can be locally overridden: `nix registry add nixpkgs ~/my-nixpkgs` + +Flake structure: +- description -- string +- inputs -- dictionary which declares all inputs + - `inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-20.03";` +- outputs -- function which takes all of the inputs and produces a result + - `outputs = { self, nixpkgs }: { }`` + - `defaultPackage.` + - `devShell.` + - `packages.` +- + +--- +- [[Nix Derivation]] + +All of the so-called "trivial" builders are built on the assumption that a project will be built separately, and that every project lives at the root of the repo. This actually can't work in my case because I do use monorepos and put many crates into a project, and don't necessarily publish all of them to crates.io. + +- [[Crane]] +- [[buildRustPackage]] +- [[buildRustCrate]] +- [[cargo2nix]] +- [[crate2nix]] +- [[How can I interrogate a Nix builder]] + +--- +- [Nix Flakes, Part 1: An introduction and tutorial - Tweag](https://www.tweag.io/blog/2020-05-25-flakes/) +- [Building Rust WASM projects with Nix | Tom Houle's homepage](https://www.tomhoule.com/2021/building-rust-wasm-with-nix-flakes/) diff --git a/Mononix/Nix For Development.md b/Mononix/Nix For Development.md new file mode 100644 index 0000000..824e843 --- /dev/null +++ b/Mononix/Nix For Development.md @@ -0,0 +1,23 @@ +- multi-language building +- caching intermediate dependencies +- cargo2nix -- probably does what I want, but it is out of date and requires updates for each new version of Rust, unless there is a hidden API to bypass that and build new versions +- crane +- supposedly modern devops is moving more towards Nix? [Taking the pulse of infrastructure management in 2023](https://www.tweag.io/blog/2023-02-23-infrastructure-pulse-2023/) +- most build tools make their own walled gardens (cargo for nix, npm for javascript, maven or ant for Java) and are hostile to any other tool getting involved +- much of Nix has tried to co-exist with these walled gardens, or work outside of them +- rust produces .rlib files, which theoretically should be analogous to .so files, but for Rust only. I feel like these should be build outputs. +- Tried doing some experiments with rustc on the command line. Currently unclear how to include an external crate + ``` + rustc -o kifu_core.rlib --edition 2021 --crate-type rlib src/lib.rs + rustc -o gtk --edition 2021 -ltokio -L../kifu-core -lgtk -lkifu_core src/main.rs + ``` +- [[Directly calling rustc]] +- A *lot* of third-party tools depend on the Cargo.toml file. This includes linters, rust-analyzer, and so forth. +- Reasonably good chance that [buildRustCrate](https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/rust.section.md#buildrustcrate-compiling-rust-crates-using-nix-instead-of-cargo-compiling-rust-crates-using-nix-instead-of-cargo) is the ultimate tool for the job in Nix, even though it would require manually setting up each dependency. `crate2Nix` is supposed to convert a Cargo.lock into an appropriate Nix file. + +--- +- [[Nix should immediately make a person's time better]] +- [[Nix Monorepos]] +- [[Scaling Rust Builds with Bazel]] +- [[Nix Flakes]] +- [[crate2nix]] \ No newline at end of file diff --git a/Mononix/Nix Monorepos.md b/Mononix/Nix Monorepos.md new file mode 100644 index 0000000..c118414 --- /dev/null +++ b/Mononix/Nix Monorepos.md @@ -0,0 +1,23 @@ + +- [Experiment in cross-compiling](https://github.com/jraygauthier/jrg-rust-cross-experiment/blob/master/simple-static-rustup-target-windows/shell.nix) +- [Building Nix flakes from Rust workspaces - Tweag](https://www.tweag.io/blog/2022-09-22-rust-nix/) + +# Rust +## Tools + +I am building this table to evaluate tools from the perspective that I would want to use Nix as a replacement for Make in a monorepo environment. "Monorepo" in this case is a shorthand for "many crates being developed in parallel with the clients that depend upon them." + +| | Dependency cache strategy | Cross-compilation | Acceptable for Nixpkgs | +| -------------------- | ----------------------------------- | ----------------- | ---------------------- | +| [[Crane]] | One derivation for all dependencies | | | +| [[buildRustPackage]] | | | | +| [[buildRustCrate]] | | | | +| [[cargo2nix]] | | | no | +| [[crate2nix]] | | | no | +| [[naersk]] | | | | + +Nothing short of caching each dependency in a separate crate will make a tool acceptable for monorepo development. + +# A strategy + +- Each crate needs to be treated as a separate diff --git a/Mononix/Nix should immediately make a person's time better.md b/Mononix/Nix should immediately make a person's time better.md new file mode 100644 index 0000000..b8f3b0a --- /dev/null +++ b/Mononix/Nix should immediately make a person's time better.md @@ -0,0 +1,8 @@ +When bringing Nix to someone as a solution to a problem, it is critical that it immediately makes their time better. Otherwise it's just another obstacle for them to overcome. + +Here are some ideas which apply if the user already has Nix installed. +- A one-liner command that pulls down and runs a program. `nix run github:source/command` +- A good devshell + +--- +- [(14) FOSDEM 2023 - Make anyone use nix - YouTube](https://www.youtube.com/watch?v=FdxvWFDwgZw) \ No newline at end of file diff --git a/Mononix/Scaling Rust Builds with Bazel.md b/Mononix/Scaling Rust Builds with Bazel.md new file mode 100644 index 0000000..4d3cdb7 --- /dev/null +++ b/Mononix/Scaling Rust Builds with Bazel.md @@ -0,0 +1,44 @@ +- *Cargo is not a build system*. This is even in the Cargo book. But everyone treats it as such (including me). +- *Cargo downloads dependencies, compiles packages, makes them distributable packages, and uploads them to crates.io*. +- their tests must: + - build a sandbox binary for executing webassembly + - build a webassembly program + - post-process the webassembly program + - build and execute a test binary that launches the sandbox binary, sends the webassembly program to the sandbox, and interacts with the program +- this test flow requires three cargo commands, and thus cannot be represented as a single cargo directive +- they saw no improvement from using [sscache](https://github.com/mozilla/sccache) +- they tried using Nix + - *requires fine-grained derivations*: each rust package must be turned into a derivation. They used cargo2nix to get there. + - developers were uncomfortable. Only the experts could modify the build rules. + - they crashed when the deployment team insisted on not using Nix. + - their builds became unbearably slow and flaky + - *developers were using the nix shell, which diverged from the CI* +- bazel + - easy to define and wire build targets + - adding new build rules requires expertise + - build files are verbose and boring + - gracefully handles linux and macos binaries, webassembly programs, os images, docker containers, etc, etc + - has aggressive caching + - remote caching + - distributed builds + - all tests are bazel tests, and so every developer can run any test locally +- migration took months + - built a prototype. sample repository mimicked the features. + - created a bazel rule called `cargo_build`, which continued to treat Cargo as a black box. + - Bazel builds binaries from Rust directly and ignores cargo. + - Added the bazel test job as soon as they possibly could + - they then started at the bottom of the stack and swapped the Cargo file out for a BUILD file one crate at a time + - they also used a visualization to show how much progress they were making + - Cargo discovers tests automatically. Bazel has to be told explicitely. They wrote a tool that compared the cargo test output and the bazel test output to ensure that all tests made it through +- remaining problems + - they've never gotten a good replacement for cargo check + - rust-analyzer support is insufficient, so they have to keep cargo files around + - there's no replacement for cargo publish yet + - + +--- +- [Scaling Rust builds with Bazel](https://mmapped.blog/posts/17-scaling-rust-builds-with-bazel.html) + +## Bazel + +I have made several attempts at using Bazel, however the rust toolchain has always failed. I have not recorded the results of the most recent failure, but it is disheartening given that Bazel is Google's tool, and Google definitely has active Rust development happening. \ No newline at end of file diff --git a/Mononix/buildRustCrate.md b/Mononix/buildRustCrate.md new file mode 100644 index 0000000..9422b0b --- /dev/null +++ b/Mononix/buildRustCrate.md @@ -0,0 +1,3 @@ +- Dependency cache strategy: +- Cross-compilation: +- Suitable for Nixpkgs: \ No newline at end of file diff --git a/Mononix/buildRustPackage.md b/Mononix/buildRustPackage.md new file mode 100644 index 0000000..9422b0b --- /dev/null +++ b/Mononix/buildRustPackage.md @@ -0,0 +1,3 @@ +- Dependency cache strategy: +- Cross-compilation: +- Suitable for Nixpkgs: \ No newline at end of file diff --git a/Mononix/cargo2nix.md b/Mononix/cargo2nix.md new file mode 100644 index 0000000..2804b11 --- /dev/null +++ b/Mononix/cargo2nix.md @@ -0,0 +1,4 @@ +- Dependency cache strategy: +- Cross-compilation: +- Suitable for Nixpkgs: No + - The cargo.lock file is too large to be effectively reviewed \ No newline at end of file diff --git a/Mononix/crate2nix.md b/Mononix/crate2nix.md new file mode 100644 index 0000000..2804b11 --- /dev/null +++ b/Mononix/crate2nix.md @@ -0,0 +1,4 @@ +- Dependency cache strategy: +- Cross-compilation: +- Suitable for Nixpkgs: No + - The cargo.lock file is too large to be effectively reviewed \ No newline at end of file