Set up the documentation for the mononix project
This commit is contained in:
commit
c0f5a07c80
|
@ -0,0 +1 @@
|
|||
{}
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"accentColor": ""
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"backlinkInDocument": true
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"file-explorer": true,
|
||||
"global-search": true,
|
||||
"switcher": true,
|
||||
"graph": true,
|
||||
"backlink": true,
|
||||
"canvas": true,
|
||||
"outgoing-link": true,
|
||||
"tag-pane": true,
|
||||
"page-preview": true,
|
||||
"daily-notes": true,
|
||||
"templates": true,
|
||||
"note-composer": true,
|
||||
"command-palette": true,
|
||||
"slash-command": false,
|
||||
"editor-status": true,
|
||||
"bookmarks": true,
|
||||
"markdown-importer": false,
|
||||
"zk-prefixer": false,
|
||||
"random-note": false,
|
||||
"outline": true,
|
||||
"word-count": true,
|
||||
"slides": false,
|
||||
"audio-recorder": false,
|
||||
"workspaces": false,
|
||||
"file-recovery": true,
|
||||
"publish": false,
|
||||
"sync": false
|
||||
}
|
|
@ -0,0 +1,20 @@
|
|||
[
|
||||
"file-explorer",
|
||||
"global-search",
|
||||
"switcher",
|
||||
"graph",
|
||||
"backlink",
|
||||
"canvas",
|
||||
"outgoing-link",
|
||||
"tag-pane",
|
||||
"page-preview",
|
||||
"daily-notes",
|
||||
"templates",
|
||||
"note-composer",
|
||||
"command-palette",
|
||||
"editor-status",
|
||||
"bookmarks",
|
||||
"outline",
|
||||
"word-count",
|
||||
"file-recovery"
|
||||
]
|
|
@ -0,0 +1 @@
|
|||
{}
|
|
@ -0,0 +1,184 @@
|
|||
{
|
||||
"main": {
|
||||
"id": "009f4879d7d58688",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "8b2d8fed8c2820e7",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "0168bf593828da5a",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "markdown",
|
||||
"state": {
|
||||
"file": "cargo2nix.md",
|
||||
"mode": "source",
|
||||
"backlinks": true,
|
||||
"source": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "f8131571839b6055",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "markdown",
|
||||
"state": {
|
||||
"file": "Index.md",
|
||||
"mode": "source",
|
||||
"backlinks": true,
|
||||
"source": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"currentTab": 1
|
||||
}
|
||||
],
|
||||
"direction": "vertical"
|
||||
},
|
||||
"left": {
|
||||
"id": "5e13cf68a0856016",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "3efb5df25c9dd56d",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "7e1974ad240e7bbf",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "file-explorer",
|
||||
"state": {
|
||||
"sortOrder": "alphabetical"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "cdeb4c8f33227fff",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "search",
|
||||
"state": {
|
||||
"query": "",
|
||||
"matchingCase": false,
|
||||
"explainSearch": false,
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5523bc2dc5f0334c",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "bookmarks",
|
||||
"state": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300,
|
||||
"collapsed": true
|
||||
},
|
||||
"right": {
|
||||
"id": "00a3036558f626b2",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "cfbf00e95752a62c",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "fcd5a94438929e0e",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "backlink",
|
||||
"state": {
|
||||
"file": "Index.md",
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical",
|
||||
"showSearch": false,
|
||||
"searchQuery": "",
|
||||
"backlinkCollapsed": false,
|
||||
"unlinkedCollapsed": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "0d9ae37b7a736590",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "outgoing-link",
|
||||
"state": {
|
||||
"file": "Index.md",
|
||||
"linksCollapsed": false,
|
||||
"unlinkedCollapsed": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "9dffaafe6dfb655f",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "tag",
|
||||
"state": {
|
||||
"sortOrder": "frequency",
|
||||
"useHierarchy": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "5ca8ea0c5c458175",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "outline",
|
||||
"state": {
|
||||
"file": "Index.md"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300,
|
||||
"collapsed": true
|
||||
},
|
||||
"left-ribbon": {
|
||||
"hiddenItems": {
|
||||
"switcher:Open quick switcher": false,
|
||||
"graph:Open graph view": false,
|
||||
"canvas:Create new canvas": false,
|
||||
"daily-notes:Open today's daily note": false,
|
||||
"templates:Insert template": false,
|
||||
"command-palette:Open command palette": false
|
||||
}
|
||||
},
|
||||
"active": "f8131571839b6055",
|
||||
"lastOpenFiles": [
|
||||
"Nix For Development.md",
|
||||
"Nix should immediately make a person's time better.md",
|
||||
"Nix Flakes.md",
|
||||
"buildRustPackage.md",
|
||||
"buildRustCrate.md",
|
||||
"crate2nix.md",
|
||||
"cargo2nix.md",
|
||||
"How can I interrogate a Nix builder.md",
|
||||
"Index.md",
|
||||
"Nix Monorepos.md",
|
||||
"Cross-compiling.md",
|
||||
"Makefiles and Monorepos.md",
|
||||
"Mononix, C library.md",
|
||||
"Mononix, Rust library.md",
|
||||
"Scaling Rust Builds with Bazel.md",
|
||||
"Crane.md"
|
||||
]
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
- Dependency cache strategy: one derivation for all dependencies.
|
||||
- Cross-compilation:
|
||||
- Suitable for Nixpkgs:
|
||||
|
||||
Lack of per-crate caching makes this unsuitable for monorepos.
|
|
@ -0,0 +1,30 @@
|
|||
# 2023-07-09
|
||||
|
||||
I did some basic things to cross-compile to raspberry pi, and I got an executable that would theoretically run on a Pi. However, the Pi is a Nixos system, so I'll need to either create a full enclosure of my dev environment, or I will need to patchelf anything that I copy over. Patchelf is not impossible, but it's not as easy as just copying the executable from one system to another.
|
||||
|
||||
rust-toolchain:
|
||||
```
|
||||
[toolchain]
|
||||
channel = "1.68.2"
|
||||
targets = [ "wasm32-unknown-unknown", "arm-unknown-linux-gnueabihf" ]
|
||||
```
|
||||
|
||||
.cargo/config.toml:
|
||||
```
|
||||
[target.arm-unknown-linux-gnueabihf]
|
||||
linker = "armv6l-unknown-linux-gnueabihf-gcc"
|
||||
```
|
||||
|
||||
flake.nix:
|
||||
```
|
||||
armPkgs = import nixpkgs {
|
||||
system = "x86_64-linux";
|
||||
crossSystem = pkgs.lib.systems.examples.raspberryPi;
|
||||
};
|
||||
|
||||
...
|
||||
|
||||
buildInputs = [
|
||||
armPkgs.stdenv.cc
|
||||
]
|
||||
```
|
|
@ -0,0 +1,5 @@
|
|||
How can I interrogate a Nix builder?
|
||||
|
||||
Nix derivations are often horrifyingly difficult to read. It is also difficult to figure out how to wrangle Nix into giving up any information about how it's interpreting the derivation.
|
||||
|
||||
In order to understand these tools that I'm working with, I need to be able to inject the equivalent of debug prints and have a very clear place to get the results back.
|
|
@ -0,0 +1,54 @@
|
|||
Tools for running builds using Nix in a monorepo.
|
||||
|
||||
Goal: Using one coherent build tool, I can develop multiple projects with common dependencies.
|
||||
|
||||
Requirements:
|
||||
- select the toolchains and versions thereof for the project
|
||||
- configure cross-compilation
|
||||
- allow multiple languages
|
||||
- co-exist with language ecosystems. For example, rust-analyzer needs cargo in order to understand Cargo.toml. However, it should be easier to fully build the project with Nix
|
||||
- every module is compiled into a derivation that downstream modules can import. Each module derivation is cached separately so that Nix only rebuilds the changed module.
|
||||
- Nix can generate a closure for an executable, even a cross-compiled one, to copy to the remote system
|
||||
|
||||
# Languages and Bindings
|
||||
|
||||
Languages:
|
||||
- C
|
||||
- Rust
|
||||
- Typescript
|
||||
- Javascript
|
||||
|
||||
Bindings:
|
||||
- C -> Rust
|
||||
- Rust -> Typescript
|
||||
- Rust -> Javascript
|
||||
|
||||
Backends
|
||||
|
||||
| | Linux x86 | Linux ARM | WASM | AVR | ESP32 |
|
||||
| ---------- | --------- | --------- | ---- | --- | ----- |
|
||||
| C | yes | yes | | yes | |
|
||||
| Rust | yes | yes | yes | yes | yes |
|
||||
| Typescript | | | yes | | |
|
||||
| Javascript | | | yes | | |
|
||||
|
||||
- [[Mononix, C library]]
|
||||
- [[Mononix, Rust library]]
|
||||
|
||||
# Build targets
|
||||
|
||||
- build
|
||||
- run
|
||||
- tests
|
||||
- code coverage profiling
|
||||
- linting
|
||||
- performance testing
|
||||
|
||||
Each of these is a separate Nix command that builds or runs the desired result.
|
||||
|
||||
---
|
||||
- [[Nix Monorepos]]
|
||||
- [[Nix For Development]]
|
||||
- [[Cross-compiling]]
|
||||
- [[Scaling Rust Builds with Bazel]]
|
||||
- [[Makefiles and Monorepos]]
|
|
@ -0,0 +1,6 @@
|
|||
Old, tried and true. Make doesn't care at all about dependency management, so I'm going to try a bunch of Makefiles to at least kick off language-native toolchains. I may need more sophisticated rules to make make library/application dependencies work well within the repository.
|
||||
|
||||
Drawbacks:
|
||||
|
||||
- I don't know how to create what amounts to a standard makefile with regular entry points
|
||||
- There's still a lot of glue that I have to work out for each language.
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
Exports:
|
||||
- Header files
|
||||
- Native DLL
|
||||
- Native static library
|
||||
- Dependencies
|
|
@ -0,0 +1,12 @@
|
|||
|
||||
Exports:
|
||||
- Rust libaries
|
||||
- Native DLL
|
||||
- Native static library
|
||||
- Dependencies
|
||||
|
||||
A Rust library can have both native system dependencies (in the form of other libraries) and Rust dependencies (in the form of Rust crates).
|
||||
|
||||
A Rust library can, with effort, be published with C bindings.
|
||||
|
||||
[[Are all Rust DLLs and static libraries meant for import to C?]]
|
|
@ -0,0 +1,39 @@
|
|||
3 part tutorial by Eelco Dolstra
|
||||
|
||||
- solves the problem where build inputs are dependent on environment variables, the nix channels, and so forth. These are inconsistent across systems.
|
||||
- *no standard way to compose nix projects*
|
||||
- packages can provide their own `flake.nix` file
|
||||
- `nix shell github:edolstra/dwarffs --command dwarffs --version`
|
||||
- flakes must declare dependencies, and dependencies must be locked to specific versions
|
||||
- `nix flake metadata` will show the flake's dependencies
|
||||
- `nix flake show` will show all of the outputs
|
||||
- `defaultPackage.<system>` must be a derivation
|
||||
- flakes are specified with a URL-like syntax: `github:edolstra/dwarffs` or `git+https://github.com/NixOS/patchelf`. The flake registery maps symbolic identifiers to actual locations. `nixpkgs` is equivalent to `github:NixOS/nixpkgs`.
|
||||
- Registry can be locally overridden: `nix registry add nixpkgs ~/my-nixpkgs`
|
||||
|
||||
Flake structure:
|
||||
- description -- string
|
||||
- inputs -- dictionary which declares all inputs
|
||||
- `inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-20.03";`
|
||||
- outputs -- function which takes all of the inputs and produces a result
|
||||
- `outputs = { self, nixpkgs }: { }``
|
||||
- `defaultPackage.<system>`
|
||||
- `devShell.<system>`
|
||||
- `packages.<system>`
|
||||
-
|
||||
|
||||
---
|
||||
- [[Nix Derivation]]
|
||||
|
||||
All of the so-called "trivial" builders are built on the assumption that a project will be built separately, and that every project lives at the root of the repo. This actually can't work in my case because I do use monorepos and put many crates into a project, and don't necessarily publish all of them to crates.io.
|
||||
|
||||
- [[Crane]]
|
||||
- [[buildRustPackage]]
|
||||
- [[buildRustCrate]]
|
||||
- [[cargo2nix]]
|
||||
- [[crate2nix]]
|
||||
- [[How can I interrogate a Nix builder]]
|
||||
|
||||
---
|
||||
- [Nix Flakes, Part 1: An introduction and tutorial - Tweag](https://www.tweag.io/blog/2020-05-25-flakes/)
|
||||
- [Building Rust WASM projects with Nix | Tom Houle's homepage](https://www.tomhoule.com/2021/building-rust-wasm-with-nix-flakes/)
|
|
@ -0,0 +1,23 @@
|
|||
- multi-language building
|
||||
- caching intermediate dependencies
|
||||
- cargo2nix -- probably does what I want, but it is out of date and requires updates for each new version of Rust, unless there is a hidden API to bypass that and build new versions
|
||||
- crane
|
||||
- supposedly modern devops is moving more towards Nix? [Taking the pulse of infrastructure management in 2023](https://www.tweag.io/blog/2023-02-23-infrastructure-pulse-2023/)
|
||||
- most build tools make their own walled gardens (cargo for nix, npm for javascript, maven or ant for Java) and are hostile to any other tool getting involved
|
||||
- much of Nix has tried to co-exist with these walled gardens, or work outside of them
|
||||
- rust produces .rlib files, which theoretically should be analogous to .so files, but for Rust only. I feel like these should be build outputs.
|
||||
- Tried doing some experiments with rustc on the command line. Currently unclear how to include an external crate
|
||||
```
|
||||
rustc -o kifu_core.rlib --edition 2021 --crate-type rlib src/lib.rs
|
||||
rustc -o gtk --edition 2021 -ltokio -L../kifu-core -lgtk -lkifu_core src/main.rs
|
||||
```
|
||||
- [[Directly calling rustc]]
|
||||
- A *lot* of third-party tools depend on the Cargo.toml file. This includes linters, rust-analyzer, and so forth.
|
||||
- Reasonably good chance that [buildRustCrate](https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/rust.section.md#buildrustcrate-compiling-rust-crates-using-nix-instead-of-cargo-compiling-rust-crates-using-nix-instead-of-cargo) is the ultimate tool for the job in Nix, even though it would require manually setting up each dependency. `crate2Nix` is supposed to convert a Cargo.lock into an appropriate Nix file.
|
||||
|
||||
---
|
||||
- [[Nix should immediately make a person's time better]]
|
||||
- [[Nix Monorepos]]
|
||||
- [[Scaling Rust Builds with Bazel]]
|
||||
- [[Nix Flakes]]
|
||||
- [[crate2nix]]
|
|
@ -0,0 +1,23 @@
|
|||
|
||||
- [Experiment in cross-compiling](https://github.com/jraygauthier/jrg-rust-cross-experiment/blob/master/simple-static-rustup-target-windows/shell.nix)
|
||||
- [Building Nix flakes from Rust workspaces - Tweag](https://www.tweag.io/blog/2022-09-22-rust-nix/)
|
||||
|
||||
# Rust
|
||||
## Tools
|
||||
|
||||
I am building this table to evaluate tools from the perspective that I would want to use Nix as a replacement for Make in a monorepo environment. "Monorepo" in this case is a shorthand for "many crates being developed in parallel with the clients that depend upon them."
|
||||
|
||||
| | Dependency cache strategy | Cross-compilation | Acceptable for Nixpkgs |
|
||||
| -------------------- | ----------------------------------- | ----------------- | ---------------------- |
|
||||
| [[Crane]] | One derivation for all dependencies | | |
|
||||
| [[buildRustPackage]] | | | |
|
||||
| [[buildRustCrate]] | | | |
|
||||
| [[cargo2nix]] | | | no |
|
||||
| [[crate2nix]] | | | no |
|
||||
| [[naersk]] | | | |
|
||||
|
||||
Nothing short of caching each dependency in a separate crate will make a tool acceptable for monorepo development.
|
||||
|
||||
# A strategy
|
||||
|
||||
- Each crate needs to be treated as a separate
|
|
@ -0,0 +1,8 @@
|
|||
When bringing Nix to someone as a solution to a problem, it is critical that it immediately makes their time better. Otherwise it's just another obstacle for them to overcome.
|
||||
|
||||
Here are some ideas which apply if the user already has Nix installed.
|
||||
- A one-liner command that pulls down and runs a program. `nix run github:source/command`
|
||||
- A good devshell
|
||||
|
||||
---
|
||||
- [(14) FOSDEM 2023 - Make anyone use nix - YouTube](https://www.youtube.com/watch?v=FdxvWFDwgZw)
|
|
@ -0,0 +1,44 @@
|
|||
- *Cargo is not a build system*. This is even in the Cargo book. But everyone treats it as such (including me).
|
||||
- *Cargo downloads dependencies, compiles packages, makes them distributable packages, and uploads them to crates.io*.
|
||||
- their tests must:
|
||||
- build a sandbox binary for executing webassembly
|
||||
- build a webassembly program
|
||||
- post-process the webassembly program
|
||||
- build and execute a test binary that launches the sandbox binary, sends the webassembly program to the sandbox, and interacts with the program
|
||||
- this test flow requires three cargo commands, and thus cannot be represented as a single cargo directive
|
||||
- they saw no improvement from using [sscache](https://github.com/mozilla/sccache)
|
||||
- they tried using Nix
|
||||
- *requires fine-grained derivations*: each rust package must be turned into a derivation. They used cargo2nix to get there.
|
||||
- developers were uncomfortable. Only the experts could modify the build rules.
|
||||
- they crashed when the deployment team insisted on not using Nix.
|
||||
- their builds became unbearably slow and flaky
|
||||
- *developers were using the nix shell, which diverged from the CI*
|
||||
- bazel
|
||||
- easy to define and wire build targets
|
||||
- adding new build rules requires expertise
|
||||
- build files are verbose and boring
|
||||
- gracefully handles linux and macos binaries, webassembly programs, os images, docker containers, etc, etc
|
||||
- has aggressive caching
|
||||
- remote caching
|
||||
- distributed builds
|
||||
- all tests are bazel tests, and so every developer can run any test locally
|
||||
- migration took months
|
||||
- built a prototype. sample repository mimicked the features.
|
||||
- created a bazel rule called `cargo_build`, which continued to treat Cargo as a black box.
|
||||
- Bazel builds binaries from Rust directly and ignores cargo.
|
||||
- Added the bazel test job as soon as they possibly could
|
||||
- they then started at the bottom of the stack and swapped the Cargo file out for a BUILD file one crate at a time
|
||||
- they also used a visualization to show how much progress they were making
|
||||
- Cargo discovers tests automatically. Bazel has to be told explicitely. They wrote a tool that compared the cargo test output and the bazel test output to ensure that all tests made it through
|
||||
- remaining problems
|
||||
- they've never gotten a good replacement for cargo check
|
||||
- rust-analyzer support is insufficient, so they have to keep cargo files around
|
||||
- there's no replacement for cargo publish yet
|
||||
-
|
||||
|
||||
---
|
||||
- [Scaling Rust builds with Bazel](https://mmapped.blog/posts/17-scaling-rust-builds-with-bazel.html)
|
||||
|
||||
## Bazel
|
||||
|
||||
I have made several attempts at using Bazel, however the rust toolchain has always failed. I have not recorded the results of the most recent failure, but it is disheartening given that Bazel is Google's tool, and Google definitely has active Rust development happening.
|
|
@ -0,0 +1,3 @@
|
|||
- Dependency cache strategy:
|
||||
- Cross-compilation:
|
||||
- Suitable for Nixpkgs:
|
|
@ -0,0 +1,3 @@
|
|||
- Dependency cache strategy:
|
||||
- Cross-compilation:
|
||||
- Suitable for Nixpkgs:
|
|
@ -0,0 +1,4 @@
|
|||
- Dependency cache strategy:
|
||||
- Cross-compilation:
|
||||
- Suitable for Nixpkgs: No
|
||||
- The cargo.lock file is too large to be effectively reviewed
|
|
@ -0,0 +1,4 @@
|
|||
- Dependency cache strategy:
|
||||
- Cross-compilation:
|
||||
- Suitable for Nixpkgs: No
|
||||
- The cargo.lock file is too large to be effectively reviewed
|
Loading…
Reference in New Issue