Vendor dependencies

Let's see how I like this workflow.

vendor/cxx/tools/buck/prelude/python/compile.bzl

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load(
+    ":manifest.bzl",
+    "ManifestInfo",  # @unused Used as a type
+)
+load(":toolchain.bzl", "PythonToolchainInfo")
+
+def compile_manifests(
+        ctx: "context",
+        manifests: [ManifestInfo.type],
+        ignore_errors: bool.type = False) -> "artifact":
+    output = ctx.actions.declare_output("bytecode")
+    cmd = cmd_args(ctx.attrs._python_toolchain[PythonToolchainInfo].host_interpreter)
+    cmd.add(ctx.attrs._python_toolchain[PythonToolchainInfo].compile)
+    cmd.add(cmd_args(output.as_output(), format = "--output={}"))
+    if ignore_errors:
+        cmd.add("--ignore-errors")
+    for manifest in manifests:
+        cmd.add(manifest.manifest)
+        cmd.hidden(manifest.artifacts)
+    ctx.actions.run(
+        cmd,
+        # On some platforms (e.g. linux), python hash code randomness can cause
+        # the bytecode to be non-deterministic, so pin via the `PYTHONHASHSEED`
+        # env var.
+        env = {"PYTHONHASHSEED": "7"},
+        category = "py_compile",
+    )
+    return output

vendor/cxx/tools/buck/prelude/python/cxx_python_extension.bzl

@@ -0,0 +1,270 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load(
+    "@prelude//cxx:cxx.bzl",
+    "get_srcs_with_flags",
+)
+load("@prelude//cxx:cxx_context.bzl", "get_cxx_toolchain_info")
+load(
+    "@prelude//cxx:cxx_library.bzl",
+    "cxx_library_parameterized",
+)
+load(
+    "@prelude//cxx:cxx_library_utility.bzl",
+    "cxx_attr_deps",
+)
+load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo")
+load(
+    "@prelude//cxx:cxx_types.bzl",
+    "CxxRuleConstructorParams",
+    "CxxRuleProviderParams",
+    "CxxRuleSubTargetParams",
+)
+load("@prelude//cxx:headers.bzl", "cxx_get_regular_cxx_headers_layout")
+load(
+    "@prelude//cxx:omnibus.bzl",
+    "explicit_roots_enabled",
+    "get_roots",
+)
+load(
+    "@prelude//linking:link_groups.bzl",
+    "merge_link_group_lib_info",
+)
+load(
+    "@prelude//linking:link_info.bzl",
+    "LinkStyle",
+    "Linkage",
+    "create_merged_link_info",
+)
+load(
+    "@prelude//linking:linkable_graph.bzl",
+    "AnnotatedLinkableRoot",
+    "create_linkable_graph",
+    "create_linkable_graph_node",
+    "create_linkable_node",
+)
+load(
+    "@prelude//linking:linkables.bzl",
+    "LinkableProviders",
+    "linkables",
+)
+load(
+    "@prelude//linking:shared_libraries.bzl",
+    "SharedLibrariesTSet",
+    "SharedLibraryInfo",
+    "merge_shared_libraries",
+)
+load("@prelude//python:toolchain.bzl", "PythonPlatformInfo", "get_platform_attr")
+load("@prelude//utils:utils.bzl", "expect", "flatten", "value_or")
+load(":manifest.bzl", "create_manifest_for_source_map")
+load(
+    ":native_python_util.bzl",
+    "merge_cxx_extension_info",
+    "rewrite_static_symbols",
+)
+load(":python.bzl", "PythonLibraryInfo")
+load(":python_library.bzl", "create_python_library_info", "dest_prefix", "gather_dep_libraries", "qualify_srcs")
+
+# This extension is basically cxx_library, plus base_module.
+# So we augment with default attributes so it has everything cxx_library has, and then call cxx_library_parameterized and work from that.
+def cxx_python_extension_impl(ctx: "context") -> ["provider"]:
+    providers = []
+
+    sub_targets = CxxRuleSubTargetParams(
+        argsfiles = True,
+        compilation_database = True,
+        headers = False,
+        link_group_map = False,
+        link_style_outputs = False,
+        xcode_data = False,
+    )
+
+    cxx_providers = CxxRuleProviderParams(
+        compilation_database = True,
+        default = False,  # We need to do some postprocessing to make sure the shared library is our default output
+        java_packaging_info = False,
+        linkable_graph = False,  # We create this here so we can correctly apply exclusions
+        link_style_outputs = False,
+        merged_native_link_info = False,
+        omnibus_root = True,
+        preprocessors = False,
+        resources = True,
+        shared_libraries = False,
+        template_placeholders = False,
+        preprocessor_for_tests = False,
+    )
+
+    impl_params = CxxRuleConstructorParams(
+        build_empty_so = True,
+        rule_type = "cxx_python_extension",
+        headers_layout = cxx_get_regular_cxx_headers_layout(ctx),
+        srcs = get_srcs_with_flags(ctx),
+        use_soname = False,
+        generate_providers = cxx_providers,
+        generate_sub_targets = sub_targets,
+        is_omnibus_root = explicit_roots_enabled(ctx),
+    )
+
+    cxx_library_info = cxx_library_parameterized(ctx, impl_params)
+    libraries = cxx_library_info.all_outputs
+    shared_output = libraries.outputs[LinkStyle("shared")]
+
+    shared_objects = libraries.solibs.values()
+    expect(len(shared_objects) == 1, "Expected exactly 1 so for cxx_python_extension: {}".format(ctx.label))
+    extension = shared_objects[0]
+
+    providers.append(DefaultInfo(
+        default_outputs = [shared_output.default],
+        other_outputs = shared_output.other,
+        sub_targets = cxx_library_info.sub_targets,
+    ))
+
+    module_name = value_or(ctx.attrs.module_name, ctx.label.name)
+    name = module_name + ".so"
+    cxx_deps = [dep for dep in cxx_attr_deps(ctx)]
+
+    linkable_providers = None
+    shared_libraries = []
+    extension_artifacts = {}
+    python_module_names = {}
+
+    # For python_cxx_extensions we need to mangle the symbol names in order to avoid collisions
+    # when linking into the main binary
+    static_output = None
+    if ctx.attrs.allow_embedding:
+        static_output = libraries.outputs[LinkStyle("static")]
+
+    if static_output != None:
+        qualified_name = dest_prefix(ctx.label, ctx.attrs.base_module)
+        if not ctx.attrs.allow_suffixing:
+            link_infos = libraries.libraries
+            pyinit_symbol = "PyInit_{}".format(module_name)
+        else:
+            suffix = qualified_name.replace("/", "_") + module_name
+            static_pic_output = libraries.outputs[LinkStyle("static_pic")]
+            cxx_toolchain = get_cxx_toolchain_info(ctx)
+            link_infos = rewrite_static_symbols(
+                ctx,
+                suffix,
+                pic_objects = static_pic_output.object_files,
+                non_pic_objects = static_output.object_files,
+                libraries = libraries.libraries,
+                cxx_toolchain = cxx_toolchain,
+            )
+            pyinit_symbol = "PyInit_{}_{}".format(module_name, suffix)
+
+        if qualified_name != "":
+            lines = ["# auto generated stub\n"]
+            stub_name = module_name + ".empty_stub"
+            extension_artifacts.update(qualify_srcs(ctx.label, ctx.attrs.base_module, {stub_name: ctx.actions.write(stub_name, lines)}))
+
+        python_module_names[qualified_name.replace("/", ".") + module_name] = pyinit_symbol
+        link_deps = linkables(cxx_deps)
+        linkable_providers = LinkableProviders(
+            link_group_lib_info = merge_link_group_lib_info(deps = cxx_deps),
+            linkable_graph = create_linkable_graph(
+                ctx = ctx,
+                node = create_linkable_graph_node(
+                    ctx = ctx,
+                    linkable_node = create_linkable_node(
+                        ctx = ctx,
+                        deps = cxx_deps,
+                        preferred_linkage = Linkage("static"),
+                        link_infos = link_infos,
+                    ),
+                ),
+                children = [d.linkable_graph for d in link_deps],
+            ),
+            merged_link_info = create_merged_link_info(
+                ctx = ctx,
+                link_infos = link_infos,
+                preferred_linkage = Linkage("static"),
+                deps = [d.merged_link_info for d in link_deps],
+            ),
+            shared_library_info = merge_shared_libraries(
+                actions = ctx.actions,
+                deps = [d.shared_library_info for d in link_deps],
+            ),
+            #linkable_root_info = field([LinkableRootInfo.type, None], None),
+        )
+
+    else:
+        # If we cannot link this extension statically we need to include it's shared libraries
+        shared_library_infos = filter(None, [x.get(SharedLibraryInfo) for x in cxx_deps])
+        shared_libraries.append(ctx.actions.tset(
+            SharedLibrariesTSet,
+            children = filter(
+                None,
+                [dep.set for dep in shared_library_infos],
+            ),
+        ))
+        extension_artifacts.update(qualify_srcs(ctx.label, ctx.attrs.base_module, {name: extension.output}))
+
+    providers.append(merge_cxx_extension_info(
+        actions = ctx.actions,
+        deps = cxx_deps,
+        linkable_providers = linkable_providers,
+        shared_libraries = shared_libraries,
+        artifacts = extension_artifacts,
+        python_module_names = python_module_names,
+    ))
+    providers.extend(cxx_library_info.providers)
+
+    # If a type stub was specified, create a manifest for export.
+    src_type_manifest = None
+    if ctx.attrs.type_stub != None:
+        src_type_manifest = create_manifest_for_source_map(
+            ctx,
+            "type_stub",
+            qualify_srcs(
+                ctx.label,
+                ctx.attrs.base_module,
+                {module_name + ".pyi": ctx.attrs.type_stub},
+            ),
+        )
+
+    # Export library info.
+    python_platform = ctx.attrs._python_toolchain[PythonPlatformInfo]
+    cxx_platform = ctx.attrs._cxx_toolchain[CxxPlatformInfo]
+    raw_deps = (
+        [ctx.attrs.deps] +
+        get_platform_attr(python_platform, cxx_platform, ctx.attrs.platform_deps)
+    )
+    deps, shared_deps = gather_dep_libraries(raw_deps)
+    providers.append(create_python_library_info(
+        ctx.actions,
+        ctx.label,
+        extensions = qualify_srcs(ctx.label, ctx.attrs.base_module, {name: extension}),
+        deps = deps,
+        shared_libraries = shared_deps,
+        src_types = src_type_manifest,
+    ))
+
+    # Omnibus providers
+
+    # Handle the case where C++ Python extensions depend on other C++ Python
+    # extensions, which should also be treated as roots.
+    roots = get_roots(ctx.label, [
+        dep
+        for dep in flatten(raw_deps)
+        # We only want to handle C++ Python extension deps, but not other native
+        # linkable deps like C++ libraries.
+        if PythonLibraryInfo in dep
+    ])
+    roots[ctx.label] = AnnotatedLinkableRoot(root = cxx_library_info.linkable_root)
+
+    linkable_graph = create_linkable_graph(
+        ctx,
+        node = create_linkable_graph_node(
+            ctx,
+            roots = roots,
+        ),
+        deps = flatten(raw_deps),
+    )
+    providers.append(linkable_graph)
+    return providers

vendor/cxx/tools/buck/prelude/python/interface.bzl

@@ -0,0 +1,67 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+# Input to build Python libraries and binaries (which are libraries wrapped in
+# an executable). The various functions here must returns the inputs annotated
+# below.
+PythonLibraryInterface = record(
+    # Shared libraries used by this Python library.
+    # {str.type: SharedLibraryInfo.type}
+    shared_libraries = field("function"),
+
+    # An iterator of PythonLibraryManifests objects. This is used to collect extensions.
+    # iterator of PythonLibraryManifests
+    iter_manifests = field("function"),
+
+    # A PythonLibraryManifestsInterface. This is used to convert manifests to
+    # arguments for pexing. Unlike iter_manifests this allows for more
+    # efficient calls, such as using t-sets projections.
+    # PythonLibraryManifestsInterface
+    manifests = field("function"),
+
+    # Returns whether this Python library includes hidden resources.
+    # bool
+    has_hidden_resources = field("function"),
+
+    # Converts the hidden resources in this Python library to arguments.
+    # _arglike of hidden resources
+    hidden_resources = field("function"),
+)
+
+PythonLibraryManifestsInterface = record(
+    # Returns the source manifests for this Python library.
+    # [_arglike] of source manifests
+    src_manifests = field("function"),
+
+    # Returns the files referenced by source manifests for this Python library.
+    # [_arglike] of source artifacts
+    src_artifacts = field("function"),
+
+    # Returns the source manifests for this Python library.
+    # [_arglike] of source manifests
+    src_type_manifests = field("function"),
+
+    # Returns the files referenced by source manifests for this Python library.
+    # [_arglike] of source artifacts
+    src_type_artifacts = field("function"),
+
+    # Returns the bytecode manifests for this Python library.
+    # [_arglike] of bytecode manifests
+    bytecode_manifests = field("function"),
+
+    # Returns the files referenced by bytecode manifests for this Python library.
+    # [_arglike] of bytecode artifacts
+    bytecode_artifacts = field("function"),
+
+    # Returns the resources manifests for this Python library.
+    # [_arglike] of resource manifests
+    resource_manifests = field("function"),
+
+    # Returns the files referenced by resource manifests for this Python library.
+    # [_arglike] of resource artifacts
+    resource_artifacts = field("function"),
+)

vendor/cxx/tools/buck/prelude/python/make_pex.bzl

@@ -0,0 +1,323 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+Rule for the inplace pex builder, and some utility methods for generic pex builder
+execution
+"""
+
+load("@prelude//:local_only.bzl", "package_python_locally")
+load(
+    "@prelude//linking:link_info.bzl",
+    "LinkedObject",  # @unused Used as a type
+)
+load("@prelude//utils:utils.bzl", "flatten")
+load(":interface.bzl", "PythonLibraryManifestsInterface")
+load(":manifest.bzl", "ManifestInfo")  # @unused Used as a type
+load(":toolchain.bzl", "PackageStyle", "PythonToolchainInfo")
+
+# This represents the input to the creation of a Pex. Manifests provide source
+# files, extensions are native extensions, and compile indicates whether we
+# should also include bytecode from manifests.
+PexModules = record(
+    manifests = field(PythonLibraryManifestsInterface.type),
+    extensions = field([ManifestInfo.type, None], None),
+    extra_manifests = field([ManifestInfo.type, None], None),
+    compile = field(bool.type, False),
+)
+
+# The output of pex creation. It's everything needed to make the DefaultInfo and RunInfo
+# providers.
+PexProviders = record(
+    default_outputs = ["artifact"],
+    other_outputs = ["artifact", "_arglike"],
+    sub_targets = {str.type: ["provider"]},
+    run_cmd = cmd_args.type,
+)
+
+def _srcs(srcs: [""], format = "{}") -> "cmd_args":
+    args = cmd_args()
+    for src in srcs:
+        args.add(cmd_args(src, format = format))
+    return args
+
+# TODO(nmj): Resources
+# TODO(nmj): Figure out how to harmonize these flags w/ existing make_xar
+#                 invocations. It might be perfectly reasonable to just have a wrapper
+#                 script that invokes make_xar in a slightly different way.
+def make_pex(
+        ctx: "context",
+        python_toolchain: "PythonToolchainInfo",
+        bundled_runtime: bool.type,
+        package_style: PackageStyle.type,
+        build_args: ["_arglike"],
+        pex_modules: PexModules.type,
+        shared_libraries: {str.type: (LinkedObject.type, bool.type)},
+        main_module: str.type,
+        hidden_resources: [None, "_arglike"]) -> PexProviders.type:
+    """
+    Passes a standardized set of flags to a `make_pex` binary to create a python
+    "executable".
+
+    Arguments:
+        - python_toolchain: Used to locate the PEX binaries.
+        - package_style: How to package this binary. Might be controlled by the
+          toolchain, but also by the rule.
+        - build_args: Extra arguments to pass to the PEX binary.
+        - pex_modules: Manifests for sources to package.
+        - shared_libraries: Shared libraries to link in. Mapping of soname to
+          artifact and whether they should be preloaded.
+        - main_module: the name of the module to execute when running the
+          resulting binary.
+        - hidden_resources: extra resources the binary depends on.
+    """
+
+    output = ctx.actions.declare_output("{}{}".format(ctx.attrs.name, python_toolchain.pex_extension))
+
+    standalone = package_style == PackageStyle("standalone")
+
+    runtime_files = []
+    if standalone and hidden_resources != None:
+        error_msg = _hidden_resources_error_message(ctx.label, hidden_resources)
+        fail(error_msg)
+    if hidden_resources != None:
+        runtime_files.extend(hidden_resources)
+
+    # HACK: runtime bundling is only implemented in make_par (via
+    # python_toolchain.make_pex_standalone), so pretend we're doing a standalone build
+    if bundled_runtime:
+        standalone = True
+
+    if not (standalone or
+            package_style == PackageStyle("inplace") or
+            package_style == PackageStyle("inplace_lite")):
+        fail("unsupported package style: {}".format(package_style))
+
+    symlink_tree_path = None
+    if not standalone:
+        symlink_tree_path = ctx.actions.declare_output("{}#link-tree".format(ctx.attrs.name))
+
+    modules_args, hidden = _pex_modules_args(ctx, pex_modules, {name: lib for name, (lib, _) in shared_libraries.items()}, symlink_tree_path)
+
+    bootstrap_args = _pex_bootstrap_args(
+        ctx,
+        python_toolchain.interpreter,
+        None,
+        python_toolchain.host_interpreter,
+        main_module,
+        output,
+        shared_libraries,
+        symlink_tree_path,
+        package_style,
+    )
+    bootstrap_args.add(build_args)
+
+    if standalone:
+        if python_toolchain.make_pex_standalone == None:
+            fail("Python toolchain does not provide make_pex_standalone")
+
+        # We support building _standalone_ packages locally to e.g. support fbcode's
+        # current style of build info stamping (e.g. T10696178).
+        prefer_local = package_python_locally(ctx, python_toolchain)
+
+        cmd = cmd_args(python_toolchain.make_pex_standalone)
+        cmd.add(modules_args)
+        cmd.add(bootstrap_args)
+        ctx.actions.run(cmd, prefer_local = prefer_local, category = "par", identifier = "standalone")
+
+    else:
+        hidden.append(symlink_tree_path)
+        modules = cmd_args(python_toolchain.make_pex_modules)
+        modules.add(modules_args)
+        ctx.actions.run(modules, category = "par", identifier = "modules")
+
+        bootstrap = cmd_args(python_toolchain.make_pex_inplace)
+        bootstrap.add(bootstrap_args)
+        ctx.actions.run(bootstrap, category = "par", identifier = "bootstrap")
+
+    runtime_files.extend(hidden)
+
+    run_args = []
+
+    # Windows can't run PAR directly.
+    if ctx.attrs._target_os_type == "windows":
+        run_args.append(ctx.attrs._python_toolchain[PythonToolchainInfo].interpreter)
+    run_args.append(output)
+
+    return PexProviders(
+        default_outputs = [output],
+        other_outputs = runtime_files,
+        sub_targets = {},
+        run_cmd = cmd_args(run_args).hidden(runtime_files),
+    )
+
+def _pex_bootstrap_args(
+        ctx: "context",
+        python_interpreter: "_arglike",
+        python_interpreter_flags: [None, str.type],
+        python_host_interpreter: "_arglike",
+        main_module: str.type,
+        output: "artifact",
+        shared_libraries: {str.type: (LinkedObject.type, bool.type)},
+        symlink_tree_path: [None, "artifact"],
+        package_style: PackageStyle.type) -> "cmd_args":
+    preload_libraries_path = ctx.actions.write(
+        "__preload_libraries.txt",
+        cmd_args([
+            "--preload={}".format(name)
+            for name, (_, preload) in shared_libraries.items()
+            if preload
+        ]),
+    )
+
+    cmd = cmd_args()
+    cmd.add(cmd_args(preload_libraries_path, format = "@{}"))
+    cmd.add([
+        "--python",
+        python_interpreter,
+        "--host-python",
+        python_host_interpreter,
+        "--entry-point",
+        main_module,
+    ])
+    if python_interpreter_flags:
+        cmd.add("--python-interpreter-flags", python_interpreter_flags)
+    if symlink_tree_path != None:
+        cmd.add(cmd_args(["--modules-dir", symlink_tree_path]).ignore_artifacts())
+
+    # Package style `inplace_lite` cannot be used with shared libraries
+    if package_style == PackageStyle("inplace_lite") and not shared_libraries:
+        cmd.add("--use-lite")
+    cmd.add(output.as_output())
+
+    return cmd
+
+def _pex_modules_args(
+        ctx: "context",
+        pex_modules: PexModules.type,
+        shared_libraries: {str.type: LinkedObject.type},
+        symlink_tree_path: [None, "artifact"]) -> ("cmd_args", ["_arglike"]):
+    """
+    Produces args to deal with a PEX's modules. Returns args to pass to the
+    modules builder, and artifacts the resulting modules would require at
+    runtime (this might be empty for e.g. a standalone pex).
+    """
+
+    srcs = []
+    src_artifacts = []
+
+    srcs.extend(pex_modules.manifests.src_manifests())
+    src_artifacts.extend(pex_modules.manifests.src_artifacts())
+
+    if pex_modules.extensions:
+        srcs.append(pex_modules.extensions.manifest)
+        src_artifacts.extend(pex_modules.extensions.artifacts)
+
+    if pex_modules.compile:
+        srcs.extend(pex_modules.manifests.bytecode_manifests())
+        src_artifacts.extend(pex_modules.manifests.bytecode_artifacts())
+
+    if pex_modules.extra_manifests:
+        srcs.append(pex_modules.extra_manifests.manifest)
+        src_artifacts.extend(pex_modules.extra_manifests.artifacts)
+
+    resources = pex_modules.manifests.resource_manifests()
+    resource_artifacts = pex_modules.manifests.resource_artifacts()
+
+    src_manifests_path = ctx.actions.write(
+        "__src_manifests.txt",
+        _srcs(srcs, format = "--module-manifest={}"),
+    )
+    resource_manifests_path = ctx.actions.write(
+        "__resource_manifests.txt",
+        _srcs(resources, format = "--resource-manifest={}"),
+    )
+
+    native_libraries = [s.output for s in shared_libraries.values()]
+    native_library_srcs_path = ctx.actions.write(
+        "__native_libraries___srcs.txt",
+        _srcs(native_libraries, format = "--native-library-src={}"),
+    )
+    native_library_dests_path = ctx.actions.write(
+        "__native_libraries___dests.txt",
+        ["--native-library-dest={}".format(lib) for lib in shared_libraries],
+    )
+
+    src_manifest_args = cmd_args(src_manifests_path).hidden(srcs)
+    resource_manifest_args = cmd_args(resource_manifests_path).hidden(resources)
+    native_library_srcs_args = cmd_args(native_library_srcs_path)
+
+    cmd = cmd_args()
+    cmd.add(cmd_args(src_manifest_args, format = "@{}"))
+    cmd.add(cmd_args(resource_manifest_args, format = "@{}"))
+    cmd.add(cmd_args(native_library_srcs_args, format = "@{}"))
+    cmd.add(cmd_args(native_library_dests_path, format = "@{}"))
+
+    dwp = []
+    if ctx.attrs.package_split_dwarf_dwp:
+        dwp = [s.dwp for s in shared_libraries.values() if s.dwp != None]
+        dwp_srcs_path = ctx.actions.write(
+            "__dwp___srcs.txt",
+            _srcs(dwp, format = "--dwp-src={}"),
+        )
+        dwp_dests_path = ctx.actions.write(
+            "__dwp___dests.txt",
+            ["--dwp-dest={}.dwp".format(lib) for lib, s in shared_libraries.items() if s.dwp != None],
+        )
+        dwp_srcs_args = cmd_args(dwp_srcs_path)
+        cmd.add(cmd_args(dwp_srcs_args, format = "@{}"))
+        cmd.add(cmd_args(dwp_dests_path, format = "@{}"))
+
+    if symlink_tree_path != None:
+        cmd.add(["--modules-dir", symlink_tree_path.as_output()])
+
+    # Accumulate all the artifacts we depend on. Only add them to the command
+    # if we are not going to create symlinks.
+    hidden = (
+        src_artifacts +
+        resource_artifacts +
+        native_libraries +
+        dwp +
+        flatten([lib.external_debug_info for lib in shared_libraries.values()])
+    )
+    if symlink_tree_path == None:
+        cmd.hidden(hidden)
+        hidden = []
+
+    return (cmd, hidden)
+
+def _hidden_resources_error_message(current_target: "label", hidden_resources) -> str.type:
+    """
+    Friendlier error message about putting non-python resources into standalone bins
+    """
+    owner_to_artifacts = {}
+
+    for resource_set in hidden_resources:
+        for resources in resource_set.traverse():
+            for r in resources:
+                if r.is_source:
+                    # Source files; do a string repr so that we get the
+                    # package path in there too
+                    owner_to_artifacts.setdefault("", []).append(str(r))
+                else:
+                    owner_to_artifacts.setdefault(r.owner, []).append(r.short_path)
+
+    msg = (
+        "Cannot package hidden srcs/resources in a standalone python_binary. " +
+        'Eliminate resources in non-Python dependencies of this python binary, use `package_style = "inplace"`, ' +
+        'use `strip_mode="full"` or turn off Split DWARF `-c fbcode.split-dwarf=false` on C++ binary resources.\n'
+    )
+
+    for (rule, resources) in owner_to_artifacts.items():
+        if rule != "":
+            msg += "Hidden srcs/resources for {}\n".format(rule)
+        else:
+            msg += "Source files:\n"
+            msg += "Find the reason this file was included with `buck2 cquery 'allpaths({}, owner(%s))' <file paths>`\n".format(current_target.raw_target())
+        for resource in sorted(resources):
+            msg += "  {}\n".format(resource)
+    return msg

vendor/cxx/tools/buck/prelude/python/manifest.bzl

@@ -0,0 +1,88 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+# Manifests are files containing information how to map sources into a package.
+# The files are JSON lists with an entry per source, where each source is 3-tuple
+# of relative destination path, artifact path, and a short description of the
+# origin of this source (used for error messages in tooling that uses these).
+ManifestInfo = record(
+    # The actual manifest file (in the form of a JSON file).
+    manifest = field("artifact"),
+    # All artifacts that are referenced in the manifest.
+    artifacts = field(["artifact"]),
+)
+
+def _write_manifest(
+        ctx: "context",
+        name: str.type,
+        entries: [(str.type, "artifact", str.type)]) -> "artifact":
+    """
+    Serialize the given source manifest entries to a JSON file.
+    """
+    return ctx.actions.write_json(name + ".manifest", entries)
+
+def create_manifest_for_entries(
+        ctx: "context",
+        name: str.type,
+        entries: [(str.type, "artifact", str.type)]) -> ManifestInfo.type:
+    """
+    Generate a source manifest for the given list of sources.
+    """
+    return ManifestInfo(
+        manifest = _write_manifest(ctx, name, entries),
+        artifacts = [a for _, a, _ in entries],
+    )
+
+def create_manifest_for_source_map(
+        ctx: "context",
+        param: str.type,
+        srcs: {str.type: "artifact"}) -> ManifestInfo.type:
+    """
+    Generate a source manifest for the given map of sources from the given rule.
+    """
+    origin = "{} {}".format(ctx.label.raw_target(), param)
+    return create_manifest_for_entries(
+        ctx,
+        param,
+        [(dest, artifact, origin) for dest, artifact in srcs.items()],
+    )
+
+def create_manifest_for_source_dir(
+        ctx: "context",
+        param: str.type,
+        extracted: "artifact") -> ManifestInfo.type:
+    """
+    Generate a source manifest for the given directory of sources from the given
+    rule.
+    """
+    manifest = ctx.actions.declare_output(param + ".manifest")
+    cmd = cmd_args(ctx.attrs._create_manifest_for_source_dir[RunInfo])
+    cmd.add("--origin={}".format(ctx.label.raw_target()))
+    cmd.add(cmd_args(manifest.as_output(), format = "--output={}"))
+    cmd.add(extracted)
+    ctx.actions.run(cmd, category = "py_source_manifest", identifier = param)
+    return ManifestInfo(manifest = manifest, artifacts = [extracted])
+
+def create_manifest_for_extensions(
+        ctx: "context",
+        extensions: {str.type: ("_a", "label")},
+        # Whether to include DWP files.
+        dwp: bool.type = False) -> ManifestInfo.type:
+    entries = []
+    for dest, (lib, label) in extensions.items():
+        entries.append((dest, lib.output, str(label.raw_target())))
+        if dwp and lib.dwp != None:
+            entries.append((dest + ".dwp", lib.dwp, str(label.raw_target()) + ".dwp"))
+    manifest = create_manifest_for_entries(ctx, "extensions", entries)
+
+    # Include external debug paths, even though they're not explicitly listed
+    # in the manifest, as python packaging may also consume debug paths which
+    # were referenced in native code.
+    for (lib, _) in extensions.values():
+        manifest.artifacts.extend(lib.external_debug_info)
+
+    return manifest

vendor/cxx/tools/buck/prelude/python/native_python_util.bzl

@@ -0,0 +1,212 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//:paths.bzl", "paths")
+load(
+    "@prelude//linking:link_info.bzl",
+    "LinkInfo",
+    "LinkInfos",
+    "LinkStyle",
+    "ObjectsLinkable",
+)
+load(
+    "@prelude//linking:linkables.bzl",
+    "LinkableProviders",  # @unused Used as type
+)
+load("@prelude//linking:shared_libraries.bzl", "SharedLibrariesTSet")
+load("@prelude//linking:strip.bzl", "strip_debug_info")
+
+LinkableProvidersTSet = transitive_set()
+
+# Info required to link cxx_python_extensions into native python binaries
+CxxExtensionLinkInfo = provider(
+    fields = [
+        "linkable_providers",  # LinkableProvidersTSet.type
+        "shared_libraries",  # SharedLibrariesTSet.type
+        "artifacts",  # {str.type: "_a"}
+        "python_module_names",  # {str.type: str.type}
+    ],
+)
+
+def merge_cxx_extension_info(
+        actions: "actions",
+        deps: ["dependency"],
+        linkable_providers: [LinkableProviders.type, None] = None,
+        shared_libraries: [SharedLibrariesTSet.type] = [],
+        artifacts: {str.type: "_a"} = {},
+        python_module_names: {str.type: str.type} = {}) -> CxxExtensionLinkInfo.type:
+    linkable_provider_children = []
+    shared_libraries = list(shared_libraries)
+    artifacts = dict(artifacts)
+    python_module_names = dict(python_module_names)
+    for dep in deps:
+        cxx_extension_info = dep.get(CxxExtensionLinkInfo)
+        if cxx_extension_info == None:
+            continue
+        linkable_provider_children.append(cxx_extension_info.linkable_providers)
+        shared_libraries.append(cxx_extension_info.shared_libraries)
+        artifacts.update(cxx_extension_info.artifacts)
+        python_module_names.update(cxx_extension_info.python_module_names)
+    linkable_providers_kwargs = {}
+    if linkable_providers != None:
+        linkable_providers_kwargs["value"] = linkable_providers
+    linkable_providers_kwargs["children"] = linkable_provider_children
+    return CxxExtensionLinkInfo(
+        linkable_providers = actions.tset(LinkableProvidersTSet, **linkable_providers_kwargs),
+        shared_libraries = actions.tset(SharedLibrariesTSet, children = shared_libraries),
+        artifacts = artifacts,
+        python_module_names = python_module_names,
+    )
+
+def rewrite_static_symbols(
+        ctx: "context",
+        suffix: str.type,
+        pic_objects: ["artifact"],
+        non_pic_objects: ["artifact"],
+        libraries: {LinkStyle.type: LinkInfos.type},
+        cxx_toolchain: "CxxToolchainInfo") -> {LinkStyle.type: LinkInfos.type}:
+    symbols_file = write_syms_file(ctx, pic_objects + non_pic_objects, suffix, cxx_toolchain)
+    static_objects, stripped_static_objects = suffix_symbols(ctx, suffix, non_pic_objects, symbols_file, cxx_toolchain)
+    static_pic_objects, stripped_static_pic_objects = suffix_symbols(ctx, suffix, pic_objects, symbols_file, cxx_toolchain)
+
+    static_info = libraries[LinkStyle("static")].default
+    updated_static_info = LinkInfo(
+        name = static_info.name,
+        pre_flags = static_info.pre_flags,
+        post_flags = static_info.post_flags,
+        linkables = [static_objects],
+        use_link_groups = static_info.use_link_groups,
+    )
+    updated_stripped_static_info = LinkInfo(
+        name = static_info.name,
+        pre_flags = static_info.pre_flags,
+        post_flags = static_info.post_flags,
+        linkables = [stripped_static_objects],
+        use_link_groups = static_info.use_link_groups,
+    )
+
+    static_pic_info = libraries[LinkStyle("static")].default
+    updated_static_pic_info = LinkInfo(
+        name = static_pic_info.name,
+        pre_flags = static_pic_info.pre_flags,
+        post_flags = static_pic_info.post_flags,
+        linkables = [static_pic_objects],
+        use_link_groups = static_pic_info.use_link_groups,
+    )
+    updated_stripped_static_pic_info = LinkInfo(
+        name = static_pic_info.name,
+        pre_flags = static_pic_info.pre_flags,
+        post_flags = static_pic_info.post_flags,
+        linkables = [stripped_static_pic_objects],
+        use_link_groups = static_pic_info.use_link_groups,
+    )
+    updated_libraries = {
+        LinkStyle("static"): LinkInfos(default = updated_static_info, stripped = updated_stripped_static_info),
+        LinkStyle("static_pic"): LinkInfos(default = updated_static_pic_info, stripped = updated_stripped_static_pic_info),
+    }
+    return updated_libraries
+
+def write_syms_file(
+        ctx: "context",
+        objects: ["artifact"],
+        suffix: str.type,
+        cxx_toolchain: "CxxToolchainInfo") -> "artifact":
+    """
+    Take a list of objects and append a suffix to all  defined symbols.
+    """
+    nm = cxx_toolchain.binary_utilities_info.nm
+    symbols_file = ctx.actions.declare_output(ctx.label.name + "_renamed_syms")
+    objects_args = cmd_args()
+    for obj in objects:
+        objects_args.add(cmd_args(obj, format = "{}"))
+
+    script_env = {
+        "NM": nm,
+        "OBJECTS": objects_args,
+        "SYMSFILE": symbols_file.as_output(),
+    }
+
+    # Compile symbols defined by all object files into a de-duplicated list of symbols to rename
+    # --no-sort tells nm not to sort the output because we are sorting it to dedupe anyway
+    # --defined-only prints only the symbols defined by this extension this way we won't rename symbols defined externally e.g. PyList_GetItem, etc...
+    # -j print only the symbol name
+    # sort -u sorts the combined list of symbols and removes any duplicate entries
+    # using awk we format the symbol names 'PyInit_hello' followed by the symbol name with the suffix appended to create the input file for objcopy
+    # objcopy uses a list of symbol name followed by updated name e.g. 'PyInit_hello PyInit_hello_package_module'
+    script = (
+        "set -euo pipefail; " +  # fail if any command in the script fails
+        '"$NM" --no-sort --defined-only -j $OBJECTS | sort -u |' +
+        ' awk \'{{print $1" "$1"_{suffix}"}}\' > '.format(suffix = suffix) +
+        '"$SYMSFILE";'
+    )
+    ctx.actions.run(
+        [
+            "/bin/bash",
+            "-c",
+            script,
+        ],
+        env = script_env,
+        category = "write_syms_file",
+        identifier = "{}_write_syms_file".format(symbols_file.basename),
+    )
+    return symbols_file
+
+def suffix_symbols(
+        ctx: "context",
+        suffix: str.type,
+        objects: ["artifact"],
+        symbols_file: "artifact",
+        cxx_toolchain: "CxxToolchainInfo") -> (ObjectsLinkable.type, ObjectsLinkable.type):
+    """
+    Take a list of objects and append a suffix to all  defined symbols.
+    """
+    objcopy = cxx_toolchain.binary_utilities_info.objcopy
+
+    artifacts = []
+    stripped_artifacts = []
+    for obj in objects:
+        base, name = paths.split_extension(obj.short_path)
+        updated_name = "_".join([base, suffix, name])
+        artifact = ctx.actions.declare_output(updated_name)
+
+        script_env = {
+            "OBJCOPY": objcopy,
+            "ORIGINAL": obj,
+            "OUT": artifact.as_output(),
+            "SYMSFILE": symbols_file,
+        }
+
+        script = (
+            "set -euo pipefail; " +  # fail if any command in the script fails
+            '"$OBJCOPY" --redefine-syms="$SYMSFILE" "$ORIGINAL" "$OUT"'  # using objcopy we pass in the symbols file to re-write the original symbol name to the now suffixed version
+        )
+
+        # Usage: objcopy [option(s)] in-file [out-file]
+        ctx.actions.run(
+            [
+                "/bin/bash",
+                "-c",
+                script,
+            ],
+            env = script_env,
+            category = "suffix_symbols",
+            identifier = updated_name,
+        )
+
+        artifacts.append(artifact)
+        updated_base, _ = paths.split_extension(artifact.short_path)
+        stripped_artifacts.append(strip_debug_info(ctx, updated_base + ".stripped.o", artifact))
+
+    default = ObjectsLinkable(
+        objects = artifacts,
+        linker_type = cxx_toolchain.linker_info.type,
+    )
+    stripped = ObjectsLinkable(
+        objects = stripped_artifacts,
+        linker_type = cxx_toolchain.linker_info.type,
+    )
+    return default, stripped

vendor/cxx/tools/buck/prelude/python/needed_coverage.bzl

@@ -0,0 +1,56 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//utils:utils.bzl", "expect")
+
+# All modules owned by a library. This will be used by top-level tests to find
+# paths that corresponds to the library.
+PythonNeededCoverageInfo = provider(fields = [
+    "modules",  # {str.type: str.type}
+])
+
+PythonNeededCoverage = record(
+    # A value from 0.0 to 1.0 indicating the ratio of coveraged code in the
+    # associated modules.
+    ratio = field(float.type),
+    # Modules that need to be covered.
+    modules = field([str.type]),
+)
+
+def _parse_python_needed_coverage_spec(
+        raw_spec: (int.type, "dependency", [str.type, None])) -> PythonNeededCoverage.type:
+    ratio_percentage, dep, specific_module = raw_spec
+
+    if ratio_percentage < 0 or ratio_percentage > 100:
+        fail("ratio_percentage must be between 0 and 100 (inclusive): {}".format(ratio_percentage))
+    ratio_percentage = ratio_percentage / 100.0
+
+    coverage = dep[PythonNeededCoverageInfo]
+    expect(coverage != None, "{} doesn't have a `PythonNeededCoverageInfo` provider", dep.label)
+
+    # Extract modules for this dep.
+    if specific_module != None:
+        module = coverage.modules.get(specific_module)
+        if module == None:
+            fail(
+                "module {} specified in needed_coverage not found in target {}"
+                    .format(specific_module, dep.label),
+            )
+        modules = [module]
+    else:
+        modules = coverage.modules.values()
+
+    expect(len(modules) > 0, "no modules found for {} ({})", dep.label, coverage)
+
+    return PythonNeededCoverage(
+        ratio = ratio_percentage,
+        modules = modules,
+    )
+
+def parse_python_needed_coverage_specs(
+        raw_specs: [(int.type, "dependency", [str.type, None])]) -> [PythonNeededCoverage.type]:
+    return [_parse_python_needed_coverage_spec(raw_spec) for raw_spec in raw_specs]

vendor/cxx/tools/buck/prelude/python/prebuilt_python_library.bzl

@@ -0,0 +1,77 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load(
+    "@prelude//:resources.bzl",
+    "ResourceInfo",
+    "gather_resources",
+)
+load(
+    "@prelude//cxx:omnibus.bzl",
+    "get_excluded",
+    "get_roots",
+)
+load(
+    "@prelude//linking:linkable_graph.bzl",
+    "create_linkable_graph",
+    "create_linkable_graph_node",
+)
+load(":compile.bzl", "compile_manifests")
+load(":manifest.bzl", "create_manifest_for_source_dir")
+load(
+    ":python_library.bzl",
+    "create_python_library_info",
+    "gather_dep_libraries",
+)
+load(":source_db.bzl", "create_source_db_no_deps_from_manifest")
+
+def prebuilt_python_library_impl(ctx: "context") -> ["provider"]:
+    providers = []
+
+    # Extract prebuilt wheel and wrap in python library provider.
+    # TODO(nmj): Make sure all attrs are used if necessary, esp compile
+    extracted_src = ctx.actions.declare_output("{}_extracted".format(ctx.label.name))
+    ctx.actions.run([ctx.attrs._extract[RunInfo], ctx.attrs.binary_src, "--output", extracted_src.as_output()], category = "py_extract_prebuilt_library")
+    deps, shared_deps = gather_dep_libraries([ctx.attrs.deps])
+    src_manifest = create_manifest_for_source_dir(ctx, "binary_src", extracted_src)
+    library_info = create_python_library_info(
+        ctx.actions,
+        ctx.label,
+        srcs = src_manifest,
+        src_types = src_manifest,
+        bytecode = create_manifest_for_source_dir(
+            ctx,
+            "bytecode",
+            compile_manifests(ctx, [src_manifest]),
+        ),
+        deps = deps,
+        shared_libraries = shared_deps,
+    )
+    providers.append(library_info)
+
+    # Create, augment and provide the linkable graph.
+    linkable_graph = create_linkable_graph(
+        ctx,
+        node = create_linkable_graph_node(
+            ctx,
+            roots = get_roots(ctx.label, ctx.attrs.deps),
+            excluded = get_excluded(deps = ctx.attrs.deps if ctx.attrs.exclude_deps_from_merged_linking else []),
+        ),
+        deps = ctx.attrs.deps,
+    )
+    providers.append(linkable_graph)
+
+    sub_targets = {"source-db-no-deps": [create_source_db_no_deps_from_manifest(ctx, src_manifest), library_info]}
+    providers.append(DefaultInfo(default_outputs = [ctx.attrs.binary_src], sub_targets = sub_targets))
+
+    # C++ resources.
+    providers.append(ResourceInfo(resources = gather_resources(
+        label = ctx.label,
+        deps = ctx.attrs.deps,
+    )))
+
+    return providers

vendor/cxx/tools/buck/prelude/python/python.bzl

@@ -0,0 +1,127 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo")
+load("@prelude//linking:shared_libraries.bzl", "traverse_shared_library_info")
+load("@prelude//utils:utils.bzl", "flatten")
+load(":interface.bzl", "PythonLibraryInterface", "PythonLibraryManifestsInterface")
+load(":manifest.bzl", "ManifestInfo")
+load(":toolchain.bzl", "PythonPlatformInfo", "get_platform_attr")
+
+PythonLibraryManifests = record(
+    label = field("label"),
+    srcs = field([ManifestInfo.type, None]),
+    src_types = field([ManifestInfo.type, None], None),
+    resources = field([(ManifestInfo.type, ["_arglike"]), None]),
+    bytecode = field([ManifestInfo.type, None]),
+    # A map of module name to to source artifact for Python extensions.
+    extensions = field([{str.type: "_a"}, None]),
+)
+
+def _bytecode_artifacts(value: PythonLibraryManifests.type):
+    if value.bytecode == None:
+        return []
+    return value.bytecode.artifacts
+
+def _bytecode_manifests(value: PythonLibraryManifests.type):
+    if value.bytecode == None:
+        return []
+    return value.bytecode.manifest
+
+def _hidden_resources(value: PythonLibraryManifests.type):
+    if value.resources == None:
+        return []
+    return value.resources[1]
+
+def _has_hidden_resources(children: [bool.type], value: [PythonLibraryManifests.type, None]):
+    if value:
+        if value.resources and len(value.resources[1]) > 0:
+            return True
+    return any(children)
+
+def _resource_manifests(value: PythonLibraryManifests.type):
+    if value.resources == None:
+        return []
+    return value.resources[0].manifest
+
+def _resource_artifacts(value: PythonLibraryManifests.type):
+    if value.resources == None:
+        return []
+    return value.resources[0].artifacts
+
+def _source_manifests(value: PythonLibraryManifests.type):
+    if value.srcs == None:
+        return []
+    return value.srcs.manifest
+
+def _source_artifacts(value: PythonLibraryManifests.type):
+    if value.srcs == None:
+        return []
+    return value.srcs.artifacts
+
+def _source_type_manifests(value: PythonLibraryManifests.type):
+    if value.src_types == None:
+        return []
+    return value.src_types.manifest
+
+def _source_type_artifacts(value: PythonLibraryManifests.type):
+    if value.src_types == None:
+        return []
+    return value.src_types.artifacts
+
+PythonLibraryManifestsTSet = transitive_set(
+    args_projections = {
+        "bytecode_artifacts": _bytecode_artifacts,
+        "bytecode_manifests": _bytecode_manifests,
+        "hidden_resources": _hidden_resources,
+        "resource_artifacts": _resource_artifacts,
+        "resource_manifests": _resource_manifests,
+        "source_artifacts": _source_artifacts,
+        "source_manifests": _source_manifests,
+        "source_type_artifacts": _source_type_artifacts,
+        "source_type_manifests": _source_type_manifests,
+    },
+    reductions = {
+        "has_hidden_resources": _has_hidden_resources,
+    },
+)
+
+# Information about a python library and its dependencies.
+# TODO(nmj): Resources in general, and mapping of resources to new paths too.
+PythonLibraryInfo = provider(fields = [
+    "manifests",  # PythonLibraryManifestsTSet
+    "shared_libraries",  # "SharedLibraryInfo"
+])
+
+def info_to_interface(info: PythonLibraryInfo.type) -> PythonLibraryInterface.type:
+    return PythonLibraryInterface(
+        shared_libraries = lambda: traverse_shared_library_info(info.shared_libraries),
+        iter_manifests = lambda: info.manifests.traverse(),
+        manifests = lambda: manifests_to_interface(info.manifests),
+        has_hidden_resources = lambda: info.manifests.reduce("has_hidden_resources"),
+        hidden_resources = lambda: [info.manifests.project_as_args("hidden_resources")],
+    )
+
+def manifests_to_interface(manifests: PythonLibraryManifestsTSet.type) -> PythonLibraryManifestsInterface.type:
+    return PythonLibraryManifestsInterface(
+        src_manifests = lambda: [manifests.project_as_args("source_manifests")],
+        src_artifacts = lambda: [manifests.project_as_args("source_artifacts")],
+        src_type_manifests = lambda: [manifests.project_as_args("source_manifests")],
+        src_type_artifacts = lambda: [manifests.project_as_args("source_artifacts")],
+        bytecode_manifests = lambda: [manifests.project_as_args("bytecode_manifests")],
+        bytecode_artifacts = lambda: [manifests.project_as_args("bytecode_artifacts")],
+        resource_manifests = lambda: [manifests.project_as_args("resource_manifests")],
+        resource_artifacts = lambda: [manifests.project_as_args("resource_artifacts")],
+    )
+
+def get_python_deps(ctx: "context"):
+    python_platform = ctx.attrs._python_toolchain[PythonPlatformInfo]
+    cxx_platform = ctx.attrs._cxx_toolchain[CxxPlatformInfo]
+    return flatten(
+        [ctx.attrs.deps] +
+        get_platform_attr(python_platform, cxx_platform, ctx.attrs.platform_deps),
+    )

vendor/cxx/tools/buck/prelude/python/python_binary.bzl

@@ -0,0 +1,429 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//cxx:compile.bzl", "CxxSrcWithFlags")
+load("@prelude//cxx:cxx.bzl", "get_cxx_auto_link_group_specs")
+load("@prelude//cxx:cxx_executable.bzl", "cxx_executable")
+load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo")
+load(
+    "@prelude//cxx:cxx_types.bzl",
+    "CxxRuleConstructorParams",
+)
+load("@prelude//cxx:headers.bzl", "cxx_get_regular_cxx_headers_layout")
+load("@prelude//cxx:link_groups.bzl", "get_link_group_info")
+load(
+    "@prelude//cxx:omnibus.bzl",
+    "all_deps",
+    "create_omnibus_libraries",
+    "get_excluded",
+    "get_omnibus_graph",
+    "get_roots",
+)
+load(
+    "@prelude//cxx:preprocessor.bzl",
+    "CPreprocessor",
+    "cxx_inherited_preprocessor_infos",
+)
+load(
+    "@prelude//linking:link_info.bzl",
+    "Linkage",
+)
+load(
+    "@prelude//linking:linkable_graph.bzl",
+    "create_linkable_graph",
+)
+load(
+    "@prelude//linking:linkables.bzl",
+    "linkables",
+)
+load(
+    "@prelude//utils:types.bzl",
+    "unchecked",  # @unused Used as a type
+)
+load("@prelude//utils:utils.bzl", "expect", "flatten", "value_or")
+load("@prelude//paths.bzl", "paths")
+load("@prelude//resources.bzl", "gather_resources")
+load(":compile.bzl", "compile_manifests")
+load(
+    ":interface.bzl",
+    "PythonLibraryInterface",  # @unused Used as a type
+)
+load(":make_pex.bzl", "PexModules", "PexProviders", "make_pex")
+load(
+    ":manifest.bzl",
+    "create_manifest_for_extensions",
+    "create_manifest_for_source_dir",
+    "create_manifest_for_source_map",
+)
+load(":native_python_util.bzl", "merge_cxx_extension_info")
+load(":python.bzl", "info_to_interface")
+load(
+    ":python_library.bzl",
+    "create_python_library_info",
+    "gather_dep_libraries",
+    "py_resources",
+    "qualify_srcs",
+)
+load(":source_db.bzl", "create_source_db", "create_source_db_no_deps")
+load(":toolchain.bzl", "NativeLinkStrategy", "PackageStyle", "PythonPlatformInfo", "PythonToolchainInfo", "get_platform_attr")
+
+OmnibusMetadataInfo = provider(fields = ["omnibus_libs", "omnibus_graph"])
+
+def _link_strategy(ctx: "context") -> NativeLinkStrategy.type:
+    if ctx.attrs.native_link_strategy != None:
+        return NativeLinkStrategy(ctx.attrs.native_link_strategy)
+    return NativeLinkStrategy(ctx.attrs._python_toolchain[PythonToolchainInfo].native_link_strategy)
+
+def _package_style(ctx: "context") -> PackageStyle.type:
+    if ctx.attrs.package_style != None:
+        return PackageStyle(ctx.attrs.package_style.lower())
+    return PackageStyle(ctx.attrs._python_toolchain[PythonToolchainInfo].package_style)
+
+# We do a lot of merging extensions, so don't use O(n) type annotations
+def _merge_extensions(
+        extensions: unchecked({str.type: ("_a", "label")}),
+        incoming_label: unchecked("label"),
+        incoming_extensions: unchecked({str.type: "_a"})) -> None:
+    """
+    Merges a incoming_extensions into `extensions`. Fails if duplicate dests exist.
+    """
+    for extension_name, incoming_artifact in incoming_extensions.items():
+        existing = extensions.get(extension_name)
+        if existing != None and existing[0] != incoming_artifact:
+            existing_artifact, existing_label = existing
+            error = (
+                "Duplicate extension: {}! Conflicting mappings:\n" +
+                "{} from {}\n" +
+                "{} from {}"
+            )
+            fail(
+                error.format(
+                    extension_name,
+                    existing_artifact,
+                    existing_label,
+                    incoming_artifact,
+                    incoming_label,
+                ),
+            )
+        extensions[extension_name] = (incoming_artifact, incoming_label)
+
+def python_executable(
+        ctx: "context",
+        main_module: str.type,
+        srcs: {str.type: "artifact"},
+        resources: {str.type: ("artifact", ["_arglike"])},
+        compile: bool.type = False) -> PexProviders.type:
+    # Returns a three tuple: the Python binary, all its potential runtime files,
+    # and a provider for its source DB.
+
+    # TODO(nmj): See if people are actually setting cxx_platform here. Really
+    #                 feels like it should be a property of the python platform
+    python_platform = ctx.attrs._python_toolchain[PythonPlatformInfo]
+    cxx_platform = ctx.attrs._cxx_toolchain[CxxPlatformInfo]
+
+    raw_deps = (
+        [ctx.attrs.deps] +
+        get_platform_attr(python_platform, cxx_platform, ctx.attrs.platform_deps)
+    )
+
+    # `preload_deps` is used later to configure `LD_PRELOAD` environment variable,
+    # here we make the actual libraries to appear in the distribution.
+    # TODO: make fully consistent with its usage later
+    raw_deps.append(ctx.attrs.preload_deps)
+    python_deps, shared_deps = gather_dep_libraries(raw_deps)
+
+    src_manifest = None
+    bytecode_manifest = None
+    if srcs:
+        src_manifest = create_manifest_for_source_map(ctx, "srcs", srcs)
+        bytecode_manifest = create_manifest_for_source_dir(
+            ctx,
+            "bytecode",
+            compile_manifests(ctx, [src_manifest]),
+        )
+
+    all_resources = {}
+    all_resources.update(resources)
+    for cxx_resources in gather_resources(ctx.label, deps = flatten(raw_deps)).values():
+        for name, resource in cxx_resources.items():
+            all_resources[paths.join("__cxx_resources__", name)] = resource
+
+    library_info = create_python_library_info(
+        ctx.actions,
+        ctx.label,
+        srcs = src_manifest,
+        resources = py_resources(ctx, all_resources) if all_resources else None,
+        bytecode = bytecode_manifest,
+        deps = python_deps,
+        shared_libraries = shared_deps,
+    )
+
+    source_db = create_source_db(ctx, src_manifest, python_deps)
+    source_db_no_deps = create_source_db_no_deps(ctx, srcs)
+
+    exe = convert_python_library_to_executable(
+        ctx,
+        main_module,
+        info_to_interface(library_info),
+        flatten(raw_deps),
+        compile,
+    )
+    exe.sub_targets.update({
+        "source-db": [source_db],
+        "source-db-no-deps": [source_db_no_deps, library_info],
+    })
+
+    return exe
+
+def convert_python_library_to_executable(
+        ctx: "context",
+        main_module: "string",
+        library: PythonLibraryInterface.type,
+        deps: ["dependency"],
+        compile: bool.type = False) -> PexProviders.type:
+    extra = {}
+
+    python_toolchain = ctx.attrs._python_toolchain[PythonToolchainInfo]
+    package_style = _package_style(ctx)
+
+    # Convert preloaded deps to a set of their names to be loaded by.
+    preload_labels = {d.label: None for d in ctx.attrs.preload_deps}
+    preload_names = {
+        name: None
+        for name, shared_lib in library.shared_libraries().items()
+        if shared_lib.label in preload_labels
+    }
+
+    extensions = {}
+    extra_manifests = None
+    for manifest in library.iter_manifests():
+        if manifest.extensions:
+            _merge_extensions(extensions, manifest.label, manifest.extensions)
+
+    # If we're using omnibus linking, re-link libraries and extensions and
+    # update the libraries we'll pull into the final binary.
+    if _link_strategy(ctx) == NativeLinkStrategy("merged"):
+        # Collect omnibus info from deps.
+        linkable_graph = create_linkable_graph(
+            ctx,
+            deps = deps,
+        )
+
+        omnibus_graph = get_omnibus_graph(
+            graph = linkable_graph,
+            # Add in any potential native root targets from our first-order deps.
+            roots = get_roots(ctx.label, deps),
+            # Exclude preloaded deps from omnibus linking, to prevent preloading
+            # the monolithic omnibus library.
+            excluded = get_excluded(deps = ctx.attrs.preload_deps),
+        )
+
+        # Link omnibus libraries.
+        omnibus_libs = create_omnibus_libraries(
+            ctx,
+            omnibus_graph,
+            ctx.attrs.linker_flags,
+            prefer_stripped_objects = ctx.attrs.prefer_stripped_native_objects,
+        )
+
+        # Extract re-linked extensions.
+        extensions = {
+            dest: (omnibus_libs.roots[label].product.shared_library, label)
+            for dest, (_, label) in extensions.items()
+        }
+        native_libs = omnibus_libs.libraries
+
+        if python_toolchain.emit_omnibus_metadata:
+            omnibus_linked_obj = omnibus_libs.omnibus
+            omnibus_info = DefaultInfo()
+            if omnibus_linked_obj:
+                omnibus_info = DefaultInfo(
+                    default_outputs = [omnibus_linked_obj.output],
+                    sub_targets = {
+                        "dwp": [DefaultInfo(default_outputs = [omnibus_linked_obj.dwp] if omnibus_linked_obj.dwp else [])],
+                    },
+                )
+            extra["omnibus"] = [
+                omnibus_info,
+                OmnibusMetadataInfo(
+                    omnibus_libs = omnibus_libs,
+                    omnibus_graph = omnibus_graph,
+                ),
+            ]
+
+            exclusion_roots = ctx.actions.write_json("omnibus/exclusion_roots.json", omnibus_libs.exclusion_roots)
+            extra["omnibus-exclusion-roots"] = [DefaultInfo(default_outputs = [exclusion_roots])]
+
+            roots = ctx.actions.write_json("omnibus/roots.json", omnibus_libs.roots)
+            extra["omnibus-roots"] = [DefaultInfo(default_outputs = [roots])]
+
+            omnibus_excluded = ctx.actions.write_json("omnibus/excluded.json", omnibus_libs.excluded)
+            extra["omnibus-excluded"] = [DefaultInfo(default_outputs = [omnibus_excluded])]
+
+            omnibus_graph_json = ctx.actions.write_json("omnibus_graph.json", omnibus_graph)
+            extra["linkable-graph"] = [DefaultInfo(default_outputs = [omnibus_graph_json])]
+    elif _link_strategy(ctx) == NativeLinkStrategy("native"):
+        expect(package_style == PackageStyle("standalone"), "native_link_strategy=native is only supported for standalone builds")
+        executable_deps = ctx.attrs.executable_deps
+        extension_info = merge_cxx_extension_info(ctx.actions, deps + executable_deps)
+        inherited_preprocessor_info = cxx_inherited_preprocessor_infos(executable_deps)
+
+        # Generate an additional C file as input
+        static_extension_info_out = ctx.actions.declare_output("static_extension_info.cpp")
+        cmd = cmd_args(python_toolchain.generate_static_extension_info[RunInfo])
+        cmd.add(cmd_args(static_extension_info_out.as_output(), format = "--output={}"))
+        cmd.add(cmd_args(["{}:{}".format(k, v) for k, v in extension_info.python_module_names.items()], format = "--extension={}"))
+
+        # TODO we don't need to do this ...
+        ctx.actions.run(cmd, category = "generate_static_extension_info")
+
+        extra["static_extension_info"] = [DefaultInfo(default_outputs = [static_extension_info_out])]
+
+        cxx_executable_srcs = [
+            CxxSrcWithFlags(file = ctx.attrs.cxx_main, flags = []),
+            CxxSrcWithFlags(file = static_extension_info_out, flags = []),
+        ]
+        extra_preprocessors = []
+        if ctx.attrs.par_style == "native":
+            extra_preprocessors.append(CPreprocessor(args = ["-DNATIVE_PAR_STYLE=1"]))
+
+        # All deps inolved in the link.
+        link_deps = (
+            linkables(executable_deps) +
+            list(extension_info.linkable_providers.traverse())
+        )
+
+        link_group_info = get_link_group_info(ctx, [d.linkable_graph for d in link_deps])
+        impl_params = CxxRuleConstructorParams(
+            rule_type = "python_binary",
+            headers_layout = cxx_get_regular_cxx_headers_layout(ctx),
+            srcs = cxx_executable_srcs,
+            extra_link_flags = ctx.attrs.linker_flags,
+            extra_preprocessors = extra_preprocessors,
+            extra_preprocessors_info = inherited_preprocessor_info,
+            extra_link_deps = link_deps,
+            exe_shared_libs_link_tree = False,
+            force_full_hybrid_if_capable = True,
+            link_group_info = link_group_info,
+            auto_link_group_specs = get_cxx_auto_link_group_specs(ctx, link_group_info),
+        )
+
+        executable_info, _, _ = cxx_executable(ctx, impl_params)
+        extra["native-executable"] = [DefaultInfo(default_outputs = [executable_info.binary])]
+
+        linkable_graph = create_linkable_graph(
+            ctx,
+            deps = deps,
+        )
+
+        # Add any shared only libs into the par
+        nodes = linkable_graph.nodes.traverse()
+        node_map = {}
+        native_libs = {}
+        shared_only = []
+        for node in filter(None, nodes):
+            if node.linkable:
+                node_map[node.label] = node.linkable
+                if node.linkable.preferred_linkage == Linkage("shared"):
+                    shared_only.append(node.label)
+        for label in all_deps(node_map, shared_only):
+            for name, shared_lib in node_map[label].shared_libs.items():
+                native_libs[name] = shared_lib
+
+        # Include shared libs from e.g. link groups.
+        native_libs.update(executable_info.shared_libs)
+
+        # Include dlopen-able shared lib deps.
+        for libs in extension_info.shared_libraries.traverse():
+            for name, shared_lib in libs.libraries.items():
+                native_libs[name] = shared_lib.lib
+
+        # Add sub-targets for libs.
+        for name, lib in native_libs.items():
+            extra[name] = [DefaultInfo(default_outputs = [lib.output])]
+
+        # TODO expect(len(executable_info.runtime_files) == 0, "OH NO THERE ARE RUNTIME FILES")
+        artifacts = dict(extension_info.artifacts)
+        artifacts["runtime/bin/{}".format(ctx.attrs.executable_name)] = executable_info.binary
+        extra_manifests = create_manifest_for_source_map(ctx, "extension_stubs", artifacts)
+        extensions = {}
+    else:
+        native_libs = {name: shared_lib.lib for name, shared_lib in library.shared_libraries().items()}
+
+    # Combine sources and extensions into a map of all modules.
+    pex_modules = PexModules(
+        manifests = library.manifests(),
+        extra_manifests = extra_manifests,
+        compile = compile,
+        extensions = create_manifest_for_extensions(
+            ctx,
+            extensions,
+            dwp = ctx.attrs.package_split_dwarf_dwp,
+        ) if extensions else None,
+    )
+
+    # Create the map of native libraries to their artifacts and whether they
+    # need to be preloaded.  Note that we merge preload deps into regular deps
+    # above, before gathering up all native libraries, so we're guaranteed to
+    # have all preload libraries (and their transitive deps) here.
+    shared_libraries = {}
+    for name, lib in native_libs.items():
+        shared_libraries[name] = lib, name in preload_names
+
+    hidden_resources = library.hidden_resources() if library.has_hidden_resources() else None
+
+    # Build the PEX.
+    pex = make_pex(
+        ctx,
+        python_toolchain,
+        ctx.attrs.bundled_runtime,
+        package_style,
+        ctx.attrs.build_args,
+        pex_modules,
+        shared_libraries,
+        main_module,
+        hidden_resources,
+    )
+
+    pex.sub_targets.update(extra)
+
+    return pex
+
+def python_binary_impl(ctx: "context") -> ["provider"]:
+    main_module = ctx.attrs.main_module
+    if ctx.attrs.main_module != None and ctx.attrs.main != None:
+        fail("Only one of main_module or main may be set. Prefer main_module as main is considered deprecated")
+    elif ctx.attrs.main != None:
+        base_module = ctx.attrs.base_module
+        if base_module == None:
+            base_module = ctx.label.package.replace("/", ".")
+        if base_module != "":
+            base_module += "."
+        main_module = base_module + ctx.attrs.main.short_path.replace("/", ".")
+        if main_module.endswith(".py"):
+            main_module = main_module[:-3]
+
+    srcs = {}
+    if ctx.attrs.main != None:
+        srcs[ctx.attrs.main.short_path] = ctx.attrs.main
+    srcs = qualify_srcs(ctx.label, ctx.attrs.base_module, srcs)
+
+    pex = python_executable(
+        ctx,
+        main_module,
+        srcs,
+        {},
+        compile = value_or(ctx.attrs.compile, False),
+    )
+    return [
+        DefaultInfo(
+            default_outputs = pex.default_outputs,
+            other_outputs = pex.other_outputs,
+            sub_targets = pex.sub_targets,
+        ),
+        RunInfo(pex.run_cmd),
+    ]

vendor/cxx/tools/buck/prelude/python/python_library.bzl

@@ -0,0 +1,349 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//:paths.bzl", "paths")
+load(
+    "@prelude//:resources.bzl",
+    "ResourceInfo",
+    "gather_resources",
+)
+load("@prelude//cxx:cxx_link_utility.bzl", "shared_libs_symlink_tree_name")
+load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo")
+load(
+    "@prelude//cxx:omnibus.bzl",
+    "get_excluded",
+    "get_roots",
+)
+load(
+    "@prelude//linking:link_info.bzl",
+    "LinkedObject",  # @unused Used as a type
+)
+load(
+    "@prelude//linking:linkable_graph.bzl",
+    "create_linkable_graph",
+    "create_linkable_graph_node",
+)
+load("@prelude//linking:shared_libraries.bzl", "SharedLibraryInfo", "merge_shared_libraries")
+load("@prelude//python:toolchain.bzl", "PythonPlatformInfo", "get_platform_attr")
+load("@prelude//utils:utils.bzl", "expect", "flatten", "from_named_set")
+load(":compile.bzl", "compile_manifests")
+load(
+    ":manifest.bzl",
+    "ManifestInfo",  # @unused Used as a type
+    "create_manifest_for_source_dir",
+    "create_manifest_for_source_map",
+)
+load(
+    ":native_python_util.bzl",
+    "merge_cxx_extension_info",
+)
+load(":needed_coverage.bzl", "PythonNeededCoverageInfo")
+load(":python.bzl", "PythonLibraryInfo", "PythonLibraryManifests", "PythonLibraryManifestsTSet")
+load(":source_db.bzl", "create_source_db", "create_source_db_no_deps")
+
+def dest_prefix(label: "label", base_module: [None, str.type]) -> str.type:
+    """
+    Find the prefix to use for placing files inside of the python link tree
+
+    This uses the label's package path if `base_module` is `None`, or `base_module`,
+    with '.' replaced by '/', if not None. If non-empty, the returned prefix will
+    end with a '/'
+    """
+    if base_module == None:
+        prefix = label.package
+    else:
+        prefix = base_module.replace(".", "/")
+
+    # Add a leading slash if we need to, but don't do that for an empty base_module
+    if prefix != "":
+        prefix += "/"
+
+    return prefix
+
+def qualify_srcs(
+        label: "label",
+        base_module: [None, str.type],
+        srcs: {str.type: "_a"}) -> {str.type: "_a"}:
+    """
+    Fully qualify package-relative sources with the rule's base module.
+
+    Arguments:
+        label: The label for the `python_library`. Used for errors, and to construct
+               the path for each source file
+        base_module: If provided, the module to prefix all files from `srcs` with in
+                     the eventual binary. If `None`, use the package path.
+                     Usage of this is discouraged, because it makes on-disk paths
+                     not match the module in execution.
+        srcs: A dictionary of {relative destination path: source file}. The derived
+              base module will be prepended to the destination.
+    """
+    prefix = dest_prefix(label, base_module)
+
+    # Use `path.normalize` here in case items in `srcs` contains relative paths.
+    return {paths.normalize(prefix + dest): src for dest, src in srcs.items()}
+
+def create_python_needed_coverage_info(
+        label: "label",
+        base_module: [None, str.type],
+        srcs: [str.type]) -> PythonNeededCoverageInfo.type:
+    prefix = dest_prefix(label, base_module)
+    return PythonNeededCoverageInfo(
+        modules = {src: prefix + src for src in srcs},
+    )
+
+def create_python_library_info(
+        actions: "actions",
+        label: "label",
+        srcs: [ManifestInfo.type, None] = None,
+        src_types: [ManifestInfo.type, None] = None,
+        bytecode: [ManifestInfo.type, None] = None,
+        resources: [(ManifestInfo.type, ["_arglike"]), None] = None,
+        extensions: [{str.type: LinkedObject.type}, None] = None,
+        deps: ["PythonLibraryInfo"] = [],
+        shared_libraries: ["SharedLibraryInfo"] = []):
+    """
+    Create a `PythonLibraryInfo` for a set of sources and deps
+
+    Arguments:
+        label: The label for the `python_library`. Used for errors, and to construct
+               the path for each source file
+        srcs: A dictionary of {relative destination path: source file}.
+        resources: A dictionary of {relative destination path: source file}.
+        prebuilt_libraries: Prebuilt python libraries to include.
+        deps: A list of `PythonLibraryInfo` objects from dependencies. These are merged
+              into the resulting `PythonLibraryInfo`, as python needs all files present
+              in the end
+    Return:
+        A fully merged `PythonLibraryInfo` provider, or fails if deps and/or srcs
+        have destination paths that collide.
+    """
+
+    manifests = PythonLibraryManifests(
+        label = label,
+        srcs = srcs,
+        src_types = src_types,
+        resources = resources,
+        bytecode = bytecode,
+        extensions = extensions,
+    )
+
+    new_shared_libraries = merge_shared_libraries(
+        actions,
+        deps = shared_libraries + [dep.shared_libraries for dep in deps],
+    )
+
+    return PythonLibraryInfo(
+        manifests = actions.tset(PythonLibraryManifestsTSet, value = manifests, children = [dep.manifests for dep in deps]),
+        shared_libraries = new_shared_libraries,
+    )
+
+def gather_dep_libraries(raw_deps: [["dependency"]]) -> (["PythonLibraryInfo"], ["SharedLibraryInfo"]):
+    """
+    Takes a list of raw dependencies, and partitions them into python_library / shared library providers.
+    Fails if a dependency is not one of these.
+    """
+    deps = []
+    shared_libraries = []
+    for raw in raw_deps:
+        for dep in raw:
+            if PythonLibraryInfo in dep:
+                deps.append(dep[PythonLibraryInfo])
+            elif SharedLibraryInfo in dep:
+                shared_libraries.append(dep[SharedLibraryInfo])
+            else:
+                # TODO(nmj): This is disabled for the moment because of:
+                #                 - the 'genrule-hack' rules that are added as deps
+                #                   on third-party whls. Not quite sure what's up
+                #                   there, but shouldn't be necessary on v2.
+                #                   (e.g. fbsource//third-party/pypi/zstandard:0.12.0-genrule-hack)
+                #fail("Dependency {} is neither a python_library, nor a prebuilt_python_library".format(dep.label))
+                pass
+    return (deps, shared_libraries)
+
+def _exclude_deps_from_omnibus(
+        ctx: "context",
+        srcs: {str.type: "artifact"}) -> bool.type:
+    # User-specified parameter.
+    if ctx.attrs.exclude_deps_from_merged_linking:
+        return True
+
+    # In some cases, Python library rules package prebuilt native extensions,
+    # in which case, we can't support library merging (since we can't re-link
+    # these extensions against new libraries).
+    for src in srcs:
+        # TODO(agallagher): Ideally, we'd prevent sources with these suffixes
+        # and requires specifying them another way to make this easier to detect.
+        if paths.split_extension(src)[1] in (".so", ".dll", ".pyd"):
+            return True
+
+    return False
+
+def _attr_srcs(ctx: "context") -> {str.type: "artifact"}:
+    python_platform = ctx.attrs._python_toolchain[PythonPlatformInfo]
+    cxx_platform = ctx.attrs._cxx_toolchain[CxxPlatformInfo]
+    all_srcs = {}
+    all_srcs.update(from_named_set(ctx.attrs.srcs))
+    for srcs in get_platform_attr(python_platform, cxx_platform, ctx.attrs.platform_srcs):
+        all_srcs.update(from_named_set(srcs))
+    return all_srcs
+
+def _attr_resources(ctx: "context") -> {str.type: ["dependency", "artifact"]}:
+    python_platform = ctx.attrs._python_toolchain[PythonPlatformInfo]
+    cxx_platform = ctx.attrs._cxx_toolchain[CxxPlatformInfo]
+    all_resources = {}
+    all_resources.update(from_named_set(ctx.attrs.resources))
+    for resources in get_platform_attr(python_platform, cxx_platform, ctx.attrs.platform_resources):
+        all_resources.update(from_named_set(resources))
+    return all_resources
+
+def py_attr_resources(ctx: "context") -> {str.type: ("artifact", ["_arglike"])}:
+    """
+    Return the resources provided by this rule, as a map of resource name to
+    a tuple of the resource artifact and any "other" outputs exposed by it.
+    """
+
+    resources = {}
+
+    for name, resource in _attr_resources(ctx).items():
+        if type(resource) == "artifact":
+            # If this is a artifact, there are no "other" artifacts.
+            other = []
+        else:
+            # Otherwise, this is a dependency, so extract the resource and other
+            # resources from the `DefaultInfo` provider.
+            info = resource[DefaultInfo]
+            expect(
+                len(info.default_outputs) == 1,
+                "expected exactly one default output from {} ({})"
+                    .format(resource, info.default_outputs),
+            )
+            [resource] = info.default_outputs
+            other = info.other_outputs
+        resources[name] = (resource, other)
+
+    return resources
+
+def py_resources(
+        ctx: "context",
+        resources: {str.type: ("artifact", ["_arglike"])}) -> (ManifestInfo.type, ["_arglike"]):
+    """
+    Generate a manifest to wrap this rules resources.
+    """
+    d = {name: resource for name, (resource, _) in resources.items()}
+    hidden = []
+    for name, (resource, other) in resources.items():
+        for o in other:
+            if type(o) == "artifact" and o.basename == shared_libs_symlink_tree_name(resource):
+                # Package the binary's shared libs next to the binary
+                # (the path is stored in RPATH relative to the binary).
+                d[paths.join(paths.dirname(name), o.basename)] = o
+            else:
+                hidden.append(o)
+    manifest = create_manifest_for_source_map(ctx, "resources", d)
+    return manifest, dedupe(hidden)
+
+def _src_types(srcs: {str.type: "artifact"}, type_stubs: {str.type: "artifact"}) -> {str.type: "artifact"}:
+    src_types = {}
+
+    # First, add all `.py` files.
+    for name, src in srcs.items():
+        base, ext = paths.split_extension(name)
+        if ext == ".py" or ext == ".pyi":
+            src_types[name] = src
+
+    # Override sources which have a corresponding type stub.
+    for name, src in type_stubs.items():
+        base, ext = paths.split_extension(name)
+        expect(ext == ".pyi", "type stubs must have `.pyi` suffix: {}", name)
+        src_types.pop(base + ".py", None)
+        src_types[name] = src
+
+    return src_types
+
+def python_library_impl(ctx: "context") -> ["provider"]:
+    # Versioned params should be intercepted and converted away via the stub.
+    expect(not ctx.attrs.versioned_srcs)
+    expect(not ctx.attrs.versioned_resources)
+
+    python_platform = ctx.attrs._python_toolchain[PythonPlatformInfo]
+    cxx_platform = ctx.attrs._cxx_toolchain[CxxPlatformInfo]
+
+    providers = []
+    sub_targets = {}
+
+    srcs = _attr_srcs(ctx)
+    qualified_srcs = qualify_srcs(ctx.label, ctx.attrs.base_module, srcs)
+    resources = qualify_srcs(ctx.label, ctx.attrs.base_module, py_attr_resources(ctx))
+    type_stubs = qualify_srcs(ctx.label, ctx.attrs.base_module, from_named_set(ctx.attrs.type_stubs))
+    src_types = _src_types(qualified_srcs, type_stubs)
+
+    src_manifest = create_manifest_for_source_map(ctx, "srcs", qualified_srcs) if qualified_srcs else None
+    src_type_manifest = create_manifest_for_source_map(ctx, "type_stubs", src_types) if src_types else None
+
+    # Compile bytecode.
+    bytecode_manifest = None
+    if src_manifest != None:
+        bytecode = compile_manifests(ctx, [src_manifest])
+        sub_targets["compile"] = [DefaultInfo(default_outputs = [bytecode])]
+        bytecode_manifest = create_manifest_for_source_dir(ctx, "bytecode", bytecode)
+
+    raw_deps = (
+        [ctx.attrs.deps] +
+        get_platform_attr(python_platform, cxx_platform, ctx.attrs.platform_deps)
+    )
+    deps, shared_libraries = gather_dep_libraries(raw_deps)
+    library_info = create_python_library_info(
+        ctx.actions,
+        ctx.label,
+        srcs = src_manifest,
+        src_types = src_type_manifest,
+        resources = py_resources(ctx, resources) if resources else None,
+        bytecode = bytecode_manifest,
+        deps = deps,
+        shared_libraries = shared_libraries,
+    )
+    providers.append(library_info)
+
+    providers.append(create_python_needed_coverage_info(ctx.label, ctx.attrs.base_module, srcs.keys()))
+
+    # Source DBs.
+    sub_targets["source-db"] = [create_source_db(ctx, src_type_manifest, deps)]
+    sub_targets["source-db-no-deps"] = [create_source_db_no_deps(ctx, src_types), library_info]
+    providers.append(DefaultInfo(sub_targets = sub_targets))
+
+    # Create, augment and provide the linkable graph.
+    deps = flatten(raw_deps)
+    linkable_graph = create_linkable_graph(
+        ctx,
+        node = create_linkable_graph_node(
+            ctx,
+            # Add in any potential native root targets from our first-order deps.
+            roots = get_roots(ctx.label, deps),
+            # Exclude preloaded deps from omnibus linking, to prevent preloading
+            # the monolithic omnibus library.
+            excluded = get_excluded(deps = (deps if _exclude_deps_from_omnibus(ctx, qualified_srcs) else [])),
+        ),
+        deps = deps,
+    )
+    providers.append(linkable_graph)
+
+    # Link info for native python
+    providers.append(
+        merge_cxx_extension_info(
+            ctx.actions,
+            deps,
+            shared_libraries = [lib_info.set for lib_info in shared_libraries if lib_info.set != None],
+        ),
+    )
+
+    # C++ resources.
+    providers.append(ResourceInfo(resources = gather_resources(
+        label = ctx.label,
+        deps = flatten(raw_deps),
+    )))
+
+    return providers

vendor/cxx/tools/buck/prelude/python/python_needed_coverage_test.bzl

@@ -0,0 +1,64 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load(
+    "@prelude//tests:re_utils.bzl",
+    "get_re_executor_from_props",
+)
+load("@prelude//test/inject_test_run_info.bzl", "inject_test_run_info")
+load(
+    ":needed_coverage.bzl",
+    "parse_python_needed_coverage_specs",
+)
+
+def python_needed_coverage_test_impl(ctx: "context") -> ["provider"]:
+    test_cmd = list(ctx.attrs.test[ExternalRunnerTestInfo].command)
+
+    test_env = {}
+    test_env.update(ctx.attrs.env)
+
+    # Pass in needed coverate flags to the test.
+    needed_coverages = parse_python_needed_coverage_specs(ctx.attrs.needed_coverage)
+    test_cmd.append("--collect-coverage")
+    test_cmd.append("--coverage-include")
+    test_cmd.append(",".join([
+        "*/{}".format(module)
+        for needed_coverage in needed_coverages
+        for module in needed_coverage.modules
+    ]))
+    for needed_coverage in needed_coverages:
+        for module in needed_coverage.modules:
+            test_cmd.append("--coverage-verdict={}={}".format(module, needed_coverage.ratio))
+
+    # A needed coverage run just runs the entire test binary as bundle to
+    # determine coverage.  Rather than special implementation in tpx, we
+    # just use a simple test type to do this, which requires settings a few
+    # additional flags/env-vars which the Python tpx test type would
+    # otherwise handle.
+    test_type = "simple"
+    test_env["TEST_PILOT"] = "1"
+
+    # Setup a RE executor based on the `remote_execution` param.
+    re_executor = get_re_executor_from_props(ctx.attrs.remote_execution)
+
+    return inject_test_run_info(
+        ctx,
+        ExternalRunnerTestInfo(
+            type = test_type,
+            command = test_cmd,
+            env = test_env,
+            labels = ctx.attrs.labels,
+            contacts = ctx.attrs.contacts,
+            default_executor = re_executor,
+            # We implicitly make this test via the project root, instead of
+            # the cell root (e.g. fbcode root).
+            run_from_project_root = re_executor != None,
+            use_project_relative_paths = re_executor != None,
+        ),
+    ) + [
+        DefaultInfo(),
+    ]

vendor/cxx/tools/buck/prelude/python/python_test.bzl

@@ -0,0 +1,83 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//:paths.bzl", "paths")
+load(
+    "@prelude//tests:re_utils.bzl",
+    "get_re_executor_from_props",
+)
+load("@prelude//utils:utils.bzl", "from_named_set", "value_or")
+load("@prelude//test/inject_test_run_info.bzl", "inject_test_run_info")
+load(":make_pex.bzl", "PexProviders")
+load(":python_binary.bzl", "python_executable")
+load(":python_library.bzl", "py_attr_resources", "qualify_srcs")
+
+def _write_test_modules_list(
+        ctx: "context",
+        srcs: {str.type: "artifact"}) -> (str.type, "artifact"):
+    """
+    Generate a python source file with a list of all test modules.
+    """
+    name = "__test_modules__.py"
+    contents = "TEST_MODULES = [\n"
+    for dst in srcs:
+        root, ext = paths.split_extension(dst)
+        if ext != ".py":
+            fail("test sources must end with .py")
+        module = root.replace("/", ".")
+        contents += "    \"{}\",\n".format(module)
+    contents += "]\n"
+    return name, ctx.actions.write(name, contents)
+
+def python_test_executable(ctx: "context") -> PexProviders.type:
+    main_module = value_or(ctx.attrs.main_module, "__test_main__")
+
+    srcs = qualify_srcs(ctx.label, ctx.attrs.base_module, from_named_set(ctx.attrs.srcs))
+
+    # Generate the test modules file and add it to sources.
+    test_modules_name, test_modules_path = _write_test_modules_list(ctx, srcs)
+    srcs[test_modules_name] = test_modules_path
+
+    # Add in default test runner.
+    srcs["__test_main__.py"] = ctx.attrs._test_main
+
+    resources = qualify_srcs(ctx.label, ctx.attrs.base_module, py_attr_resources(ctx))
+
+    return python_executable(
+        ctx,
+        main_module,
+        srcs,
+        resources,
+        compile = value_or(ctx.attrs.compile, False),
+    )
+
+def python_test_impl(ctx: "context") -> ["provider"]:
+    pex = python_test_executable(ctx)
+    test_cmd = pex.run_cmd
+
+    # Setup a RE executor based on the `remote_execution` param.
+    re_executor = get_re_executor_from_props(ctx.attrs.remote_execution)
+
+    return inject_test_run_info(
+        ctx,
+        ExternalRunnerTestInfo(
+            type = "pyunit",
+            command = [test_cmd],
+            env = ctx.attrs.env,
+            labels = ctx.attrs.labels,
+            contacts = ctx.attrs.contacts,
+            default_executor = re_executor,
+            # We implicitly make this test via the project root, instead of
+            # the cell root (e.g. fbcode root).
+            run_from_project_root = re_executor != None,
+            use_project_relative_paths = re_executor != None,
+        ),
+    ) + [DefaultInfo(
+        default_outputs = pex.default_outputs,
+        other_outputs = pex.other_outputs,
+        sub_targets = pex.sub_targets,
+    )]

vendor/cxx/tools/buck/prelude/python/source_db.bzl

@@ -0,0 +1,60 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load(
+    ":manifest.bzl",
+    "ManifestInfo",  # @unused Used as a type
+)
+load(":python.bzl", "PythonLibraryManifestsTSet")
+load(":toolchain.bzl", "PythonToolchainInfo")
+
+def create_source_db(
+        ctx: "context",
+        srcs: [ManifestInfo.type, None],
+        python_deps: ["PythonLibraryInfo"]) -> DefaultInfo.type:
+    output = ctx.actions.declare_output("db.json")
+    artifacts = []
+
+    python_toolchain = ctx.attrs._python_toolchain[PythonToolchainInfo]
+    cmd = cmd_args(python_toolchain.make_source_db)
+    cmd.add(cmd_args(output.as_output(), format = "--output={}"))
+
+    # Pass manifests for rule's sources.
+    if srcs != None:
+        cmd.add(cmd_args(srcs.manifest, format = "--sources={}"))
+        artifacts.extend(srcs.artifacts)
+
+    # Pass manifests for transitive deps.
+    dep_manifests = ctx.actions.tset(PythonLibraryManifestsTSet, children = [d.manifests for d in python_deps])
+
+    dependencies = cmd_args(dep_manifests.project_as_args("source_type_manifests"), format = "--dependency={}")
+    dependencies_file = ctx.actions.write("source_db_dependencies", dependencies)
+    dependencies_file = cmd_args(dependencies_file, format = "@{}").hidden(dependencies)
+
+    cmd.add(dependencies_file)
+    artifacts.append(dep_manifests.project_as_args("source_type_artifacts"))
+
+    ctx.actions.run(cmd, category = "py_source_db")
+
+    return DefaultInfo(default_outputs = [output], other_outputs = artifacts)
+
+def create_source_db_no_deps(
+        ctx: "context",
+        srcs: [{str.type: "artifact"}, None]) -> DefaultInfo.type:
+    content = {} if srcs == None else srcs
+    output = ctx.actions.write_json("db_no_deps.json", content)
+    return DefaultInfo(default_outputs = [output], other_outputs = content.values())
+
+def create_source_db_no_deps_from_manifest(
+        ctx: "context",
+        srcs: ManifestInfo.type) -> DefaultInfo.type:
+    output = ctx.actions.declare_output("db_no_deps.json")
+    cmd = cmd_args(ctx.attrs._python_toolchain[PythonToolchainInfo].make_source_db_no_deps)
+    cmd.add(cmd_args(output.as_output(), format = "--output={}"))
+    cmd.add(srcs.manifest)
+    ctx.actions.run(cmd, category = "py_source_db")
+    return DefaultInfo(default_outputs = [output], other_outputs = srcs.artifacts)

vendor/cxx/tools/buck/prelude/python/toolchain.bzl

@@ -0,0 +1,77 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+load("@prelude//utils:platform_flavors_util.bzl", "by_platform")
+
+# The ways that Python exectuables handle native linkable dependencies.
+NativeLinkStrategy = enum(
+    # Statically links extensions into an embedded python binary
+    "native",
+    # Pull transitive native deps in as fully linked standalone shared libraries.
+    # This is typically the fastest build-time link strategy, as it requires no
+    # top-level context and therefore can shared build artifacts with all other
+    # binaries using this strategy.
+    "separate",
+    # Statically link all transitive native deps, which don't have an explicit
+    # dep from non-C/C++ code (e.g. Python), into a monolithic shared library.
+    # Native dep roots, which have an explicit dep from non-C/C++ code, remain
+    # as fully linked standalone shared libraries so that, typically, application
+    # code doesn't need to change to work with this strategy. This strategy
+    # incurs a relatively big build-time cost, but can significantly reduce the
+    # size of native code and number of shared libraries pulled into the binary.
+    "merged",
+)
+
+PackageStyle = enum(
+    "inplace",
+    "standalone",
+    "inplace_lite",
+)
+
+PythonToolchainInfo = provider(fields = [
+    "build_standalone_binaries_locally",
+    "compile",
+    # The interpreter to use to compile bytecode.
+    "host_interpreter",
+    "interpreter",
+    "version",
+    "native_link_strategy",
+    "generate_static_extension_info",
+    "package_style",
+    "make_source_db",
+    "make_source_db_no_deps",
+    "make_pex_inplace",
+    "make_pex_standalone",
+    "make_pex_modules",
+    "pex_executor",
+    "pex_extension",
+    "emit_omnibus_metadata",
+])
+
+# Stores "platform"/flavor name used to resolve *platform_* arguments
+PythonPlatformInfo = provider(fields = [
+    "name",
+])
+
+def get_platform_attr(
+        python_platform_info: "PythonPlatformInfo",
+        cxx_platform_info: "CxxPlatformInfo",
+        xs: [(str.type, "_a")]) -> ["_a"]:
+    """
+    Take a platform_* value, and the non-platform version, and concat into a list
+    of values based on the cxx/python platform
+    """
+    python_platform = python_platform_info.name
+    cxx_platform = cxx_platform_info.name
+    return by_platform([python_platform, cxx_platform], xs)
+
+python = struct(
+    PythonToolchainInfo = PythonToolchainInfo,
+    PythonPlatformInfo = PythonPlatformInfo,
+    PackageStyle = PackageStyle,
+    NativeLinkStrategy = NativeLinkStrategy,
+)

vendor/cxx/tools/buck/prelude/python/tools/TARGETS.v2

@@ -0,0 +1,72 @@
+prelude = native
+
+prelude.python_bootstrap_binary(
+    name = "extract",
+    main = "extract.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "create_manifest_for_source_dir",
+    main = "create_manifest_for_source_dir.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "make_source_db",
+    main = "make_source_db.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "make_source_db_no_deps",
+    main = "make_source_db_no_deps.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.export_file(
+    name = "__test_main__.py",
+    src = "__test_main__.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "make_pex_inplace.py",
+    main = "make_pex_inplace.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.export_file(
+    name = "run_inplace_lite.py.in",
+    src = "run_inplace_lite.py.in",
+)
+
+prelude.export_file(
+    name = "run_inplace.py.in",
+    src = "run_inplace.py.in",
+)
+
+prelude.command_alias(
+    name = "make_pex_inplace",
+    exe = ":make_pex_inplace.py",
+    args = ["--template", "$(location :run_inplace.py.in)", "--template-lite", "$(location :run_inplace_lite.py.in)"],
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "make_pex_modules",
+    main = "make_pex_modules.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.export_file(
+    name = "compile.py",
+    src = "compile.py",
+    visibility = ["PUBLIC"],
+)
+
+prelude.python_bootstrap_binary(
+    name = "generate_static_extension_info",
+    main = "generate_static_extension_info.py",
+    visibility = ["PUBLIC"],
+)

vendor/cxx/tools/buck/prelude/python/tools/__test_main__.py

@@ -0,0 +1,789 @@
+#!/usr/local/bin/python2.6 -tt
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+This file contains the main module code for buck python test programs.
+
+By default, this is the main module for all python_test() rules.  However,
+rules can also specify their own custom main_module.  If you write your own
+main module, you can import this module as tools.test.stubs.fbpyunit, to access
+any of its code to help implement your main module.
+"""
+
+# pyre-unsafe
+
+from __future__ import print_function
+
+import contextlib
+import ctypes
+import fnmatch
+import json
+import logging
+import optparse
+import os
+import platform
+import re
+import sys
+import time
+import traceback
+import unittest
+import warnings
+
+
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+    import imp
+
+try:
+    from StringIO import StringIO  # type: ignore
+except ImportError:
+    from io import StringIO
+try:
+    import coverage  # type: ignore
+except ImportError:
+    coverage = None
+try:
+    from importlib.machinery import SourceFileLoader
+except ImportError:
+    SourceFileLoader = None
+
+
+EXIT_CODE_SUCCESS = 0
+EXIT_CODE_TEST_FAILURE = 70
+
+
+class TestStatus(object):
+
+    ABORTED = "FAILURE"
+    PASSED = "SUCCESS"
+    FAILED = "FAILURE"
+    EXPECTED_FAILURE = "SUCCESS"
+    UNEXPECTED_SUCCESS = "FAILURE"
+    SKIPPED = "ASSUMPTION_VIOLATION"
+    EXCLUDED = "EXCLUDED"
+
+
+class PathMatcher(object):
+    def __init__(self, include_patterns, omit_patterns):
+        self.include_patterns = include_patterns
+        self.omit_patterns = omit_patterns
+
+    def omit(self, path):
+        """
+        Omit iff matches any of the omit_patterns or the include patterns are
+        not empty and none is matched
+        """
+        path = os.path.realpath(path)
+        return any(fnmatch.fnmatch(path, p) for p in self.omit_patterns) or (
+            self.include_patterns
+            and not any(fnmatch.fnmatch(path, p) for p in self.include_patterns)
+        )
+
+    def include(self, path):
+        return not self.omit(path)
+
+
+class DebugWipeFinder(object):
+    """
+    PEP 302 finder that uses a DebugWipeLoader for all files which do not need
+    coverage
+    """
+
+    def __init__(self, matcher):
+        self.matcher = matcher
+
+    def find_module(self, fullname, path=None):
+        _, _, basename = fullname.rpartition(".")
+        try:
+            fd, pypath, (_, _, kind) = imp.find_module(basename, path)
+        except Exception:
+            # Finding without hooks using the imp module failed. One reason
+            # could be that there is a zip file on sys.path. The imp module
+            # does not support loading from there. Leave finding this module to
+            # the others finders in sys.meta_path.
+            return None
+
+        if hasattr(fd, "close"):
+            fd.close()
+        if kind != imp.PY_SOURCE:
+            return None
+        if self.matcher.include(pypath):
+            return None
+
+        """
+        This is defined to match CPython's PyVarObject struct
+        """
+
+        class PyVarObject(ctypes.Structure):
+            _fields_ = [
+                ("ob_refcnt", ctypes.c_long),
+                ("ob_type", ctypes.c_void_p),
+                ("ob_size", ctypes.c_ulong),
+            ]
+
+        class DebugWipeLoader(SourceFileLoader):
+            """
+            PEP302 loader that zeros out debug information before execution
+            """
+
+            def get_code(self, fullname):
+                code = super(DebugWipeLoader, self).get_code(fullname)
+                if code:
+                    # Ideally we'd do
+                    # code.co_lnotab = b''
+                    # But code objects are READONLY. Not to worry though; we'll
+                    # directly modify CPython's object
+                    code_impl = PyVarObject.from_address(id(code.co_lnotab))
+                    code_impl.ob_size = 0
+                return code
+
+        return DebugWipeLoader(fullname, pypath)
+
+
+def optimize_for_coverage(cov, include_patterns, omit_patterns):
+    """
+    We get better performance if we zero out debug information for files which
+    we're not interested in. Only available in CPython 3.3+
+    """
+    matcher = PathMatcher(include_patterns, omit_patterns)
+    if SourceFileLoader and platform.python_implementation() == "CPython":
+        sys.meta_path.insert(0, DebugWipeFinder(matcher))
+
+
+class TeeStream(object):
+    def __init__(self, *streams):
+        self._streams = streams
+
+    def write(self, data):
+        for stream in self._streams:
+            stream.write(data)
+
+    def flush(self):
+        for stream in self._streams:
+            stream.flush()
+
+    def isatty(self):
+        return False
+
+
+class CallbackStream(object):
+    def __init__(self, callback, bytes_callback=None, orig=None):
+        self._callback = callback
+        self._fileno = orig.fileno() if orig else None
+
+        # Python 3 APIs:
+        # - `encoding` is a string holding the encoding name
+        # - `errors` is a string holding the error-handling mode for encoding
+        # - `buffer` should look like an io.BufferedIOBase object
+
+        self.errors = orig.errors if orig else None
+        if bytes_callback:
+            # those members are only on the io.TextIOWrapper
+            self.encoding = orig.encoding if orig else "UTF-8"
+            self.buffer = CallbackStream(bytes_callback, orig=orig)
+
+    def write(self, data):
+        self._callback(data)
+
+    def flush(self):
+        pass
+
+    def isatty(self):
+        return False
+
+    def fileno(self):
+        return self._fileno
+
+
+# pyre-fixme[11]: Annotation `unittest._TextTestResult` is not defined as a type.
+class BuckTestResult(unittest._TextTestResult):
+    """
+    Our own TestResult class that outputs data in a format that can be easily
+    parsed by buck's test runner.
+    """
+
+    def __init__(
+        self, stream, descriptions, verbosity, show_output, main_program, suite
+    ):
+        super(BuckTestResult, self).__init__(stream, descriptions, verbosity)
+        self._main_program = main_program
+        self._suite = suite
+        self._results = []
+        self._current_test = None
+        self._saved_stdout = sys.stdout
+        self._saved_stderr = sys.stderr
+        self._show_output = show_output
+
+    def getResults(self):
+        return self._results
+
+    def startTest(self, test):
+        super(BuckTestResult, self).startTest(test)
+
+        # Pass in the real stdout and stderr filenos.  We can't really do much
+        # here to intercept callers who directly operate on these fileno
+        # objects.
+        sys.stdout = CallbackStream(
+            self.addStdout, self.addStdoutBytes, orig=sys.stdout
+        )
+        sys.stderr = CallbackStream(
+            self.addStderr, self.addStderrBytes, orig=sys.stderr
+        )
+        self._current_test = test
+        self._test_start_time = time.time()
+        self._current_status = TestStatus.ABORTED
+        self._messages = []
+        self._stacktrace = None
+        self._stdout = ""
+        self._stderr = ""
+
+    def _find_next_test(self, suite):
+        """
+        Find the next test that has not been run.
+        """
+
+        for test in suite:
+
+            # We identify test suites by test that are iterable (as is done in
+            # the builtin python test harness).  If we see one, recurse on it.
+            if hasattr(test, "__iter__"):
+                test = self._find_next_test(test)
+
+            # The builtin python test harness sets test references to `None`
+            # after they have run, so we know we've found the next test up
+            # if it's not `None`.
+            if test is not None:
+                return test
+
+    def stopTest(self, test):
+        sys.stdout = self._saved_stdout
+        sys.stderr = self._saved_stderr
+
+        super(BuckTestResult, self).stopTest(test)
+
+        # If a failure occured during module/class setup, then this "test" may
+        # actually be a `_ErrorHolder`, which doesn't contain explicit info
+        # about the upcoming test.  Since we really only care about the test
+        # name field (i.e. `_testMethodName`), we use that to detect an actual
+        # test cases, and fall back to looking the test up from the suite
+        # otherwise.
+        if not hasattr(test, "_testMethodName"):
+            test = self._find_next_test(self._suite)
+
+        self._results.append(
+            {
+                "testCaseName": "{0}.{1}".format(
+                    test.__class__.__module__, test.__class__.__name__
+                ),
+                "testCase": test._testMethodName,
+                "type": self._current_status,
+                "time": int((time.time() - self._test_start_time) * 1000),
+                "message": os.linesep.join(self._messages),
+                "stacktrace": self._stacktrace,
+                "stdOut": self._stdout,
+                "stdErr": self._stderr,
+            }
+        )
+
+        self._current_test = None
+
+    def stopTestRun(self):
+        cov = self._main_program.get_coverage()
+        if cov is not None:
+            self._results.append({"coverage": cov})
+
+    @contextlib.contextmanager
+    def _withTest(self, test):
+        self.startTest(test)
+        yield
+        self.stopTest(test)
+
+    def _setStatus(self, test, status, message=None, stacktrace=None):
+        assert test == self._current_test
+        self._current_status = status
+        self._stacktrace = stacktrace
+        if message is not None:
+            if message.endswith(os.linesep):
+                message = message[:-1]
+            self._messages.append(message)
+
+    def setStatus(self, test, status, message=None, stacktrace=None):
+        # addError() may be called outside of a test if one of the shared
+        # fixtures (setUpClass/tearDownClass/setUpModule/tearDownModule)
+        # throws an error.
+        #
+        # In this case, create a fake test result to record the error.
+        if self._current_test is None:
+            with self._withTest(test):
+                self._setStatus(test, status, message, stacktrace)
+        else:
+            self._setStatus(test, status, message, stacktrace)
+
+    def setException(self, test, status, excinfo):
+        exctype, value, tb = excinfo
+        self.setStatus(
+            test,
+            status,
+            "{0}: {1}".format(exctype.__name__, value),
+            "".join(traceback.format_tb(tb)),
+        )
+
+    def addSuccess(self, test):
+        super(BuckTestResult, self).addSuccess(test)
+        self.setStatus(test, TestStatus.PASSED)
+
+    def addError(self, test, err):
+        super(BuckTestResult, self).addError(test, err)
+        self.setException(test, TestStatus.ABORTED, err)
+
+    def addFailure(self, test, err):
+        super(BuckTestResult, self).addFailure(test, err)
+        self.setException(test, TestStatus.FAILED, err)
+
+    def addSkip(self, test, reason):
+        super(BuckTestResult, self).addSkip(test, reason)
+        self.setStatus(test, TestStatus.SKIPPED, "Skipped: %s" % (reason,))
+
+    def addExpectedFailure(self, test, err):
+        super(BuckTestResult, self).addExpectedFailure(test, err)
+        self.setException(test, TestStatus.EXPECTED_FAILURE, err)
+
+    def addUnexpectedSuccess(self, test):
+        super(BuckTestResult, self).addUnexpectedSuccess(test)
+        self.setStatus(test, TestStatus.UNEXPECTED_SUCCESS, "Unexpected success")
+
+    def addStdout(self, val):
+        self._stdout += val
+        if self._show_output:
+            self._saved_stdout.write(val)
+            self._saved_stdout.flush()
+
+    def addStdoutBytes(self, val):
+        string = val.decode("utf-8", errors="backslashreplace")
+        self.addStdout(string)
+
+    def addStderr(self, val):
+        self._stderr += val
+        if self._show_output:
+            self._saved_stderr.write(val)
+            self._saved_stderr.flush()
+
+    def addStderrBytes(self, val):
+        string = val.decode("utf-8", errors="backslashreplace")
+        self.addStderr(string)
+
+
+class BuckTestRunner(unittest.TextTestRunner):
+    def __init__(self, main_program, suite, show_output=True, **kwargs):
+        super(BuckTestRunner, self).__init__(**kwargs)
+        self.show_output = show_output
+        self._main_program = main_program
+        self._suite = suite
+
+    def _makeResult(self):
+        return BuckTestResult(
+            self.stream,
+            self.descriptions,
+            self.verbosity,
+            self.show_output,
+            self._main_program,
+            self._suite,
+        )
+
+
+def _format_test_name(test_class, attrname):
+    """
+    Format the name of the test buck-style.
+    """
+    return "{0}.{1}#{2}".format(test_class.__module__, test_class.__name__, attrname)
+
+
+class StderrLogHandler(logging.StreamHandler):
+    """
+    This class is very similar to logging.StreamHandler, except that it
+    always uses the current sys.stderr object.
+
+    StreamHandler caches the current sys.stderr object when it is constructed.
+    This makes it behave poorly in unit tests, which may replace sys.stderr
+    with a StringIO buffer during tests.  The StreamHandler will continue using
+    the old sys.stderr object instead of the desired StringIO buffer.
+    """
+
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+    @property
+    def stream(self):
+        return sys.stderr
+
+
+class RegexTestLoader(unittest.TestLoader):
+    def __init__(self, regex=None):
+        self.regex = regex
+        super(RegexTestLoader, self).__init__()
+
+    def getTestCaseNames(self, testCaseClass):
+        """
+        Return a sorted sequence of method names found within testCaseClass
+        """
+
+        testFnNames = super(RegexTestLoader, self).getTestCaseNames(testCaseClass)
+        if self.regex is None:
+            return testFnNames
+        robj = re.compile(self.regex)
+        matched = []
+        for attrname in testFnNames:
+            fullname = _format_test_name(testCaseClass, attrname)
+            if robj.search(fullname):
+                matched.append(attrname)
+        return matched
+
+
+class Loader(object):
+    def __init__(self, modules, regex=None):
+        self.modules = modules
+        self.regex = regex
+
+    def load_all(self):
+        loader = RegexTestLoader(self.regex)
+        test_suite = unittest.TestSuite()
+        for module_name in self.modules:
+            __import__(module_name, level=0)
+            module = sys.modules[module_name]
+            module_suite = loader.loadTestsFromModule(module)
+            test_suite.addTest(module_suite)
+        return test_suite
+
+    def load_args(self, args):
+        loader = RegexTestLoader(self.regex)
+
+        suites = []
+        for arg in args:
+            suite = loader.loadTestsFromName(arg)
+            # loadTestsFromName() can only process names that refer to
+            # individual test functions or modules.  It can't process package
+            # names.  If there were no module/function matches, check to see if
+            # this looks like a package name.
+            if suite.countTestCases() != 0:
+                suites.append(suite)
+                continue
+
+            # Load all modules whose name is <arg>.<something>
+            prefix = arg + "."
+            for module in self.modules:
+                if module.startswith(prefix):
+                    suite = loader.loadTestsFromName(module)
+                    suites.append(suite)
+
+        return loader.suiteClass(suites)
+
+
+class MainProgram(object):
+    """
+    This class implements the main program.  It can be subclassed by
+    users who wish to customize some parts of the main program.
+    (Adding additional command line options, customizing test loading, etc.)
+    """
+
+    DEFAULT_VERBOSITY = 2
+
+    def __init__(self, argv):
+        self.init_option_parser()
+        self.parse_options(argv)
+        self.setup_logging()
+
+    def init_option_parser(self):
+        usage = "%prog [options] [TEST] ..."
+        op = optparse.OptionParser(usage=usage, add_help_option=False)
+        self.option_parser = op
+
+        op.add_option(
+            "--hide-output",
+            dest="show_output",
+            action="store_false",
+            default=True,
+            help="Suppress data that tests print to stdout/stderr, and only "
+            "show it if the test fails.",
+        )
+        op.add_option(
+            "-o",
+            "--output",
+            help="Write results to a file in a JSON format to be read by Buck",
+        )
+        op.add_option(
+            "-f",
+            "--failfast",
+            action="store_true",
+            default=False,
+            help="Stop after the first failure",
+        )
+        op.add_option(
+            "-l",
+            "--list-tests",
+            action="store_true",
+            dest="list",
+            default=False,
+            help="List tests and exit",
+        )
+        op.add_option(
+            "-L",
+            "--list-format",
+            dest="list_format",
+            choices=["buck", "python"],
+            default="python",
+            help="List tests format",
+        )
+        op.add_option(
+            "-r",
+            "--regex",
+            default=None,
+            help="Regex to apply to tests, to only run those tests",
+        )
+        op.add_option(
+            "--collect-coverage",
+            action="store_true",
+            default=False,
+            help="Collect test coverage information",
+        )
+        op.add_option(
+            "--coverage-include",
+            default="*",
+            help='File globs to include in converage (split by ",")',
+        )
+        op.add_option(
+            "--coverage-omit",
+            default="",
+            help='File globs to omit from converage (split by ",")',
+        )
+        op.add_option(
+            "--logger",
+            action="append",
+            metavar="<category>=<level>",
+            default=[],
+            help="Configure log levels for specific logger categories",
+        )
+        op.add_option(
+            "-q",
+            "--quiet",
+            action="count",
+            default=0,
+            help="Decrease the verbosity (may be specified multiple times)",
+        )
+        op.add_option(
+            "-v",
+            "--verbosity",
+            action="count",
+            default=self.DEFAULT_VERBOSITY,
+            help="Increase the verbosity (may be specified multiple times)",
+        )
+        op.add_option(
+            "-?", "--help", action="help", help="Show this help message and exit"
+        )
+
+    def parse_options(self, argv):
+        self.options, self.test_args = self.option_parser.parse_args(argv[1:])
+        self.options.verbosity -= self.options.quiet
+
+        if self.options.collect_coverage and coverage is None:
+            self.option_parser.error("coverage module is not available")
+        self.options.coverage_include = self.options.coverage_include.split(",")
+        if self.options.coverage_omit == "":
+            self.options.coverage_omit = []
+        else:
+            self.options.coverage_omit = self.options.coverage_omit.split(",")
+
+    def setup_logging(self):
+        # Configure the root logger to log at INFO level.
+        # This is similar to logging.basicConfig(), but uses our
+        # StderrLogHandler instead of a StreamHandler.
+        fmt = logging.Formatter("%(pathname)s:%(lineno)s: %(message)s")
+        log_handler = StderrLogHandler()
+        log_handler.setFormatter(fmt)
+        root_logger = logging.getLogger()
+        root_logger.addHandler(log_handler)
+        root_logger.setLevel(logging.INFO)
+
+        level_names = {
+            "debug": logging.DEBUG,
+            "info": logging.INFO,
+            "warn": logging.WARNING,
+            "warning": logging.WARNING,
+            "error": logging.ERROR,
+            "critical": logging.CRITICAL,
+            "fatal": logging.FATAL,
+        }
+
+        for value in self.options.logger:
+            parts = value.rsplit("=", 1)
+            if len(parts) != 2:
+                self.option_parser.error(
+                    "--logger argument must be of the "
+                    "form <name>=<level>: %s" % value
+                )
+            name = parts[0]
+            level_name = parts[1].lower()
+            level = level_names.get(level_name)
+            if level is None:
+                self.option_parser.error(
+                    "invalid log level %r for log category %s" % (parts[1], name)
+                )
+            logging.getLogger(name).setLevel(level)
+
+    def create_loader(self):
+        import __test_modules__
+
+        return Loader(__test_modules__.TEST_MODULES, self.options.regex)
+
+    def load_tests(self):
+        loader = self.create_loader()
+        if self.options.collect_coverage:
+            self.start_coverage()
+            include = self.options.coverage_include
+            omit = self.options.coverage_omit
+            if include and "*" not in include:
+                optimize_for_coverage(self.cov, include, omit)
+
+        if self.test_args:
+            suite = loader.load_args(self.test_args)
+        else:
+            suite = loader.load_all()
+        if self.options.collect_coverage:
+            self.cov.start()
+        return suite
+
+    def get_tests(self, test_suite):
+        tests = []
+
+        for test in test_suite:
+            if isinstance(test, unittest.TestSuite):
+                tests.extend(self.get_tests(test))
+            else:
+                tests.append(test)
+
+        return tests
+
+    def run(self):
+        test_suite = self.load_tests()
+
+        if self.options.list:
+            for test in self.get_tests(test_suite):
+                if self.options.list_format == "python":
+                    name = str(test)
+                elif self.options.list_format == "buck":
+                    method_name = getattr(test, "_testMethodName", "")
+                    name = _format_test_name(test.__class__, method_name)
+                else:
+                    raise Exception(
+                        "Bad test list format: %s" % (self.options.list_format,)
+                    )
+
+                print(name)
+            return EXIT_CODE_SUCCESS
+        else:
+            result = self.run_tests(test_suite)
+            if self.options.output is not None:
+                with open(self.options.output, "w") as f:
+                    json.dump(result.getResults(), f, indent=4, sort_keys=True)
+            if not result.wasSuccessful():
+                return EXIT_CODE_TEST_FAILURE
+            return EXIT_CODE_SUCCESS
+
+    def run_tests(self, test_suite):
+        # Install a signal handler to catch Ctrl-C and display the results
+        # (but only if running >2.6).
+        if sys.version_info[0] > 2 or sys.version_info[1] > 6:
+            unittest.installHandler()
+
+        # Run the tests
+        runner = BuckTestRunner(
+            self,
+            test_suite,
+            verbosity=self.options.verbosity,
+            show_output=self.options.show_output,
+        )
+        result = runner.run(test_suite)
+
+        if self.options.collect_coverage and self.options.show_output:
+            self.cov.stop()
+            if self.cov.html_report:
+                self.cov.html_report()
+            else:
+                self.cov.report(file=sys.stdout)
+
+        return result
+
+    def start_coverage(self):
+        if not self.options.collect_coverage:
+            return
+
+        # Keep the original working dir in case tests use os.chdir
+        self._original_working_dir = os.getcwd()
+
+        self.cov = coverage.Coverage(
+            include=self.options.coverage_include, omit=self.options.coverage_omit
+        )
+        self.cov.erase()
+        self.cov.start()
+
+    def get_coverage(self):
+        if not self.options.collect_coverage:
+            return None
+        result = {}
+
+        # Switch back to the original working directory.
+        os.chdir(self._original_working_dir)
+
+        self.cov.stop()
+
+        try:
+            f = StringIO()
+            self.cov.report(file=f)
+            lines = f.getvalue().split("\n")
+        except coverage.misc.CoverageException:
+            # Nothing was covered. That's fine by us
+            return result
+
+        # N.B.: the format of the coverage library's output differs
+        # depending on whether one or more files are in the results
+        for line in lines[2:]:
+            if line.strip("-") == "":
+                break
+            r = line.split()[0]
+            analysis = self.cov.analysis2(r)
+            covString = self.convert_to_diff_cov_str(analysis)
+            if covString:
+                result[r] = covString
+
+        return result
+
+    def convert_to_diff_cov_str(self, analysis):
+        # Info on the format of analysis:
+        # http://nedbatchelder.com/code/coverage/api.html
+        if not analysis:
+            return None
+        numLines = max(
+            analysis[1][-1] if len(analysis[1]) else 0,
+            analysis[2][-1] if len(analysis[2]) else 0,
+            analysis[3][-1] if len(analysis[3]) else 0,
+        )
+        lines = ["N"] * numLines
+        for l in analysis[1]:
+            lines[l - 1] = "C"
+        for l in analysis[2]:
+            lines[l - 1] = "X"
+        for l in analysis[3]:
+            lines[l - 1] = "U"
+        return "".join(lines)
+
+
+def main(argv):
+    return MainProgram(sys.argv).run()
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

vendor/cxx/tools/buck/prelude/python/tools/compile.py

@@ -0,0 +1,94 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+Example usage:
+$ cat inputs.manifest
+[["foo.py", "input/foo.py", "//my_rule:foo"]]
+$ compile.py --output=out-dir --ignore-errors inputs.manifest
+$ find out-dir -type f
+out-dir/foo.pyc
+"""
+
+# pyre-unsafe
+
+import argparse
+import errno
+import json
+import os
+import py_compile
+import sys
+
+
+if sys.version_info[0] == 3:
+    import importlib
+
+    DEFAULT_FORMAT = importlib.util.cache_from_source("{pkg}/{name}.py")
+else:
+    DEFAULT_FORMAT = "{pkg}/{name}.pyc"
+
+
+def get_py_path(module):
+    return module.replace(".", os.sep) + ".py"
+
+
+def get_pyc_path(module, fmt):
+    try:
+        package, name = module.rsplit(".", 1)
+    except ValueError:
+        package, name = "", module
+    parts = fmt.split(os.sep)
+    for idx in range(len(parts)):
+        if parts[idx] == "{pkg}":
+            parts[idx] = package.replace(".", os.sep)
+        elif parts[idx].startswith("{name}"):
+            parts[idx] = parts[idx].format(name=name)
+    return os.path.join(*parts)
+
+
+def _mkdirs(dirpath):
+    try:
+        os.makedirs(dirpath)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
+    parser.add_argument("-o", "--output", required=True)
+    parser.add_argument("-f", "--format", default=DEFAULT_FORMAT)
+    parser.add_argument("-i", "--ignore-errors", action="store_true")
+    parser.add_argument("manifests", nargs="*")
+    args = parser.parse_args(argv[1:])
+
+    _mkdirs(args.output)
+
+    for manifest_path in args.manifests:
+        with open(manifest_path) as mf:
+            manifest = json.load(mf)
+        for dst, src, _ in manifest:
+            # This is going to try to turn a path into a Python module, so
+            # reduce the scope for bugs in get_pyc_path by normalizing first.
+            dst = os.path.normpath(dst)
+            # We only care about python sources.
+            base, ext = os.path.splitext(dst)
+            if ext != ".py":
+                continue
+            module = base.replace(os.sep, ".")
+            pyc = os.path.join(args.output, get_pyc_path(module, args.format))
+            _mkdirs(os.path.dirname(pyc))
+            py_compile.compile(
+                src,
+                cfile=pyc,
+                dfile=get_py_path(module),
+                doraise=not args.ignore_errors,
+                invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH,
+            )
+
+
+sys.exit(main(sys.argv))

vendor/cxx/tools/buck/prelude/python/tools/create_manifest_for_source_dir.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+import argparse
+import json
+import os
+import sys
+from typing import List
+
+
+def main(argv: List[str]) -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", type=argparse.FileType("w"), default=sys.stdout)
+    parser.add_argument("--origin", help="description of source origin")
+    parser.add_argument("--prefix", help="prefix to prepend to destinations")
+    parser.add_argument("extracted", help="path to directory of sources")
+    args = parser.parse_args(argv[1:])
+
+    entries = []
+    for root, _, files in os.walk(args.extracted):
+        for name in files:
+            path = os.path.join(root, name)
+            dest = os.path.relpath(path, args.extracted)
+            if args.prefix is not None:
+                dest = os.path.join(args.prefix, dest)
+            entry = [dest, path]
+            if args.origin is not None:
+                entry.append(args.origin)
+            entries.append(entry)
+
+    json.dump(entries, args.output, indent=2, sort_keys=True)
+
+
+sys.exit(main(sys.argv))

vendor/cxx/tools/buck/prelude/python/tools/extract.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+Quick and dirty wrapper to extract zip files; python 3.6.2+
+
+extract.py my_zip_file.zip --output=output_directory
+"""
+
+import argparse
+import shutil
+from pathlib import Path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Extract .zip files to a directory in a cross platform manner"
+    )
+    parser.add_argument(
+        "--output", type=Path, required=True, help="The directory to write to"
+    )
+    parser.add_argument("src", type=Path, help="The archive to extract to --output")
+    args = parser.parse_args()
+
+    args.output.mkdir(parents=True, exist_ok=True)
+    shutil.unpack_archive(args.src, args.output, "zip")
+
+
+if __name__ == "__main__":
+    main()

vendor/cxx/tools/buck/prelude/python/tools/generate_static_extension_info.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+import argparse
+import sys
+from typing import List
+
+
+def main(argv: List[str]) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", type=argparse.FileType("w"), default=sys.stdout)
+    parser.add_argument("--extension", action="append", default=[])
+    args = parser.parse_args(argv[1:])
+    out_file = args.output
+
+    externs = []
+    table = [
+        "struct _inittab _static_extension_info[] = {",
+    ]
+    for python_name in args.extension:
+        module_name, pyinit_func = python_name.split(":")
+        # If this is a top level module we do not suffix the PyInit_ symbol
+        externs.append(f"PyMODINIT_FUNC {pyinit_func}(void);")
+        table.append(f'  {{ "{module_name}", {pyinit_func} }},')
+    table.append("  { nullptr, nullptr },")
+    table.append("};")
+
+    out_lines = (
+        [
+            '#include "Python.h"',
+            '#include "import.h"',
+        ]
+        + externs
+        + table
+    )
+
+    for line in out_lines:
+        print(line, file=out_file)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

vendor/cxx/tools/buck/prelude/python/tools/make_pex_inplace.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+Create a bootstrapper pex for inplace python binaries
+
+This script:
+    - Writes out a bootstrapper pex script that knows where this symlink tree is,
+      and uses it, along with the provided entry point to run the python script.
+      It does this by replacing a few special strings like <MODULES_DIR> and
+      <MAIN_MODULE>
+
+A full usage might be something like this:
+
+$ cat template.in
+(see prelude/python/run_inplace_lite.py.in)
+$ ./make_pex_inplace.py  \\
+    --template prelude/python/run_inplace.py.in \\
+    # These two args create the hashbang for the bootstrapper script \\
+    --python="/usr/bin/python3" \\
+    --python-interpreter-flags="-Es" \\
+    # This is based on the path in dests. This is the module that gets executed \\
+    # to start program execution \\
+    --entry-point=lib.foo  \\
+    --output=bin.pex \\
+    # This is the symlink tree \\
+    --modules-dir=bin__link-tree
+$ ./bin.pex
+...
+"""
+
+import argparse
+import errno
+import os
+import platform
+import stat
+from pathlib import Path
+
+
+def parse_args() -> argparse.Namespace:
+    # TODO(nmj): Go back and verify all of the various flags that make_xar
+    #                 takes, and standardize on that so that the calling convention
+    #                 is the same regardless of the "make_X" binary that's used.
+    parser = argparse.ArgumentParser(
+        description=(
+            "Create a python inplace binary, writing a symlink tree to a directory, "
+            "and a bootstrapper pex file to file"
+        ),
+        fromfile_prefix_chars="@",
+    )
+    parser.add_argument(
+        "--template",
+        required=True,
+        type=Path,
+        help="The template file for the .pex bootstrapper script",
+    )
+    parser.add_argument(
+        "--template-lite",
+        required=True,
+        type=Path,
+        help="The template file for the .pex bootstrapper script, if it's simple",
+    )
+
+    parser.add_argument(
+        "--preload",
+        type=Path,
+        dest="preload_libraries",
+        action="append",
+        default=[],
+        help="A list of native libraries to add to LD_PRELOAD",
+    )
+    parser.add_argument(
+        "--python",
+        required=True,
+        help="The python binary to put in the bootstrapper hashbang",
+    )
+    parser.add_argument(
+        "--host-python",
+        required=True,
+        help="The host python binary to use to e.g. compiling bytecode",
+    )
+    parser.add_argument(
+        "--python-interpreter-flags",
+        default="-Es",
+        help="The interpreter flags for the hashbang",
+    )
+    parser.add_argument(
+        "--entry-point", required=True, help="The main module to execute"
+    )
+    parser.add_argument(
+        "--modules-dir",
+        required=True,
+        type=Path,
+        help="The link tree directory to use at runtime",
+    )
+    parser.add_argument(
+        "--use-lite",
+        help="Whether to use the lite template",
+        action="store_true",
+    )
+    parser.add_argument(
+        "output",
+        type=Path,
+        help="Where to write the bootstrapper script to",
+    )
+    parser.add_argument(
+        "--native-libs-env-var",
+        default=(
+            "DYLD_LIBRARY_PATH" if platform.system() == "Darwin" else "LD_LIBRARY_PATH"
+        ),
+        help="The dynamic loader env used to find native library deps",
+    )
+    # Compatibility with existing make_par scripts
+    parser.add_argument("--passthrough", action="append", default=[])
+
+    return parser.parse_args()
+
+
+def write_bootstrapper(args: argparse.Namespace) -> None:
+    """Write the .pex bootstrapper script using a template"""
+
+    template = args.template_lite if args.use_lite else args.template
+    with open(template, "r", encoding="utf8") as fin:
+        data = fin.read()
+
+    # Because this can be invoked from other directories, find the relative path
+    # from this .par to the modules dir, and use that.
+    relative_modules_dir = os.path.relpath(args.modules_dir, args.output.parent)
+
+    # TODO(nmj): Remove this hack. So, if arg0 in your shebang is a bash script
+    #                 (like /usr/local/fbcode/platform007/bin/python3.7 on macs is)
+    #                 OSX just sort of ignores it and tries to run your thing with
+    #                 the current shell. So, we hack in /usr/bin/env in the front
+    #                 for now, and let it do the lifting. OSX: Bringing you the best
+    #                 of 1980s BSD in 2021...
+    #                 Also, make sure we add PYTHON_INTERPRETER_FLAGS back. We had to
+    #                 exclude it for now, because linux doesn't like multiple args
+    #                 after /usr/bin/env
+
+    ld_preload = "None"
+    if args.preload_libraries:
+        ld_preload = repr(":".join(p.name for p in args.preload_libraries))
+
+    new_data = data.replace("<PYTHON>", "/usr/bin/env " + str(args.python))
+    new_data = new_data.replace("<PYTHON_INTERPRETER_FLAGS>", "")
+    # new_data = new_data.replace(
+    #    "<PYTHON_INTERPRETER_FLAGS>", args.python_interpreter_flags
+    # )
+    new_data = new_data.replace("<MODULES_DIR>", str(relative_modules_dir))
+    new_data = new_data.replace("<MAIN_MODULE>", args.entry_point)
+
+    # Things that are only required for the full template
+    new_data = new_data.replace("<NATIVE_LIBS_ENV_VAR>", args.native_libs_env_var)
+    new_data = new_data.replace("<NATIVE_LIBS_DIR>", repr(relative_modules_dir))
+    new_data = new_data.replace("<NATIVE_LIBS_PRELOAD_ENV_VAR>", "LD_PRELOAD")
+    new_data = new_data.replace("<NATIVE_LIBS_PRELOAD>", ld_preload)
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.output, "w", encoding="utf8") as fout:
+        fout.write(new_data)
+    mode = os.stat(args.output).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    os.chmod(args.output, mode)
+
+
+def main() -> None:
+    args = parse_args()
+    write_bootstrapper(args)
+
+
+if __name__ == "__main__":
+    main()

vendor/cxx/tools/buck/prelude/python/tools/make_pex_modules.py

@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+Create the link tree for inplace Python binaries.
+
+This does a few things:
+    - Allows remapping of source files (via the srcs/dests arguments) and resources.
+    - Merges extracted .whl files into the tree
+    - Adds __init__.py where needed
+    - Writes out a bootstrapper pex script that knows where this symlink tree is,
+      and uses it, along with the provided entry point to run the python script.
+      It does this by replacing a few special strings like <MODULES_DIR> and
+      <MAIN_MODULE>
+
+A full usage might be something like this:
+
+$ cat srcs
+srcs/foo.py
+srcs/bar.py
+third-party/baz.whl__extracted
+$ cat dests
+lib/foo.py
+bar.py
+.
+$ ls third-party/baz.whl__extracted
+baz/tp_foo.py
+baz/tp_bar.py
+$ cat template.in
+(see prelude/python/run_inplace_lite.py.in)
+$ ./make_pex_inplace.py  \\
+    --template prelude/python/run_inplace.py.in \\
+    --module-srcs=@srcs \\
+    --module-dests=@dests \\
+    # This is the symlink tree \\
+    --modules-dir=bin__link-tree
+$ find bin__link-tree
+lib/__init__.py
+lib/foo.py
+bar.py
+baz/tp_foo.py
+baz/tp_bar.py
+"""
+
+import argparse
+import errno
+import json
+import os
+from pathlib import Path
+from typing import Dict, Set, Tuple
+
+# Suffixes which should trigger `__init__.py` additions.
+# TODO(agallaher): This was coped from v1, but some things below probably
+# don't need to be here (e.g. `.pyd`).
+_MODULE_SUFFIXES = {
+    ".dll",
+    ".py",
+    ".pyd",
+    ".so",
+}
+
+
+def parse_args() -> argparse.Namespace:
+    # TODO(nmj): Go back and verify all of the various flags that make_xar
+    #                 takes, and standardize on that so that the calling convention
+    #                 is the same regardless of the "make_X" binary that's used.
+    parser = argparse.ArgumentParser(
+        description=(
+            "Create a python inplace binary, writing a symlink tree to a directory, "
+            "and a bootstrapper pex file to file"
+        ),
+        fromfile_prefix_chars="@",
+    )
+    parser.add_argument(
+        "--module-manifest",
+        action="append",
+        dest="module_manifests",
+        default=[],
+        help="A path to a JSON file with modules to be linked.",
+    )
+    parser.add_argument(
+        "--resource-manifest",
+        action="append",
+        dest="resource_manifests",
+        default=[],
+        help="A path to a JSON file with resources to be linked.",
+    )
+    parser.add_argument(
+        "--native-library-src",
+        type=Path,
+        dest="native_library_srcs",
+        action="append",
+        default=[],
+        help="A list of native libraries to use",
+    )
+    parser.add_argument(
+        "--native-library-dest",
+        type=Path,
+        dest="native_library_dests",
+        action="append",
+        default=[],
+        help=(
+            "A list of relative destination paths for each of the native "
+            "libraries in --native-library-src"
+        ),
+    )
+    parser.add_argument(
+        "--dwp-src",
+        type=Path,
+        dest="dwp_srcs",
+        action="append",
+        default=[],
+        help="A list of dwp for native libraries to use",
+    )
+    parser.add_argument(
+        "--dwp-dest",
+        type=Path,
+        dest="dwp_dests",
+        action="append",
+        default=[],
+        help=(
+            "A list of relative destination paths for each of the dwp for native "
+            "libraries in --dwp-src"
+        ),
+    )
+    parser.add_argument(
+        "--native-library-manifest",
+        action="append",
+        dest="native_library_manifests",
+        default=[],
+        help="A path to a JSON file with native libraries to be linked.",
+    )
+    parser.add_argument(
+        "--modules-dir",
+        required=True,
+        type=Path,
+        help="The link tree directory to write to",
+    )
+
+    return parser.parse_args()
+
+
+def add_path_mapping(
+    path_mapping: Dict[Path, Tuple[str, str]],
+    dirs_to_create: Set[Path],
+    src: Path,
+    new_dest: Path,
+    origin: str = "unknown",
+) -> None:
+    """
+    Add the mapping of a destination path into `path_mapping`, by getting the
+    relative path to the source, and making sure that there are no
+    collisions (and erroring in that case)
+    """
+
+    def format_src(src: str, origin: str) -> str:
+        out = "`{}`".format(src)
+        if origin is not None:
+            out += " (from {})".format(origin)
+        return out
+
+    link_path = os.path.relpath(src, new_dest.parent)
+    if new_dest in path_mapping:
+        prev, prev_origin = path_mapping[new_dest]
+        if prev != link_path:
+            raise ValueError(
+                "Destination path `{}` specified at both {} and {} (`{}` before relativisation)".format(
+                    new_dest,
+                    format_src(link_path, origin),
+                    format_src(prev, prev_origin),
+                    src,
+                )
+            )
+    path_mapping[new_dest] = (link_path, origin)
+    dirs_to_create.add(new_dest.parent)
+
+
+def _lexists(path: Path) -> bool:
+    """
+    Like `Path.exists()` but works on dangling. symlinks
+    """
+
+    try:
+        path.lstat()
+    except FileNotFoundError:
+        return False
+    except OSError as e:
+        if e.errno == errno.ENOENT:
+            return False
+        raise
+    return True
+
+
+def create_modules_dir(args: argparse.Namespace) -> None:
+    args.modules_dir.mkdir(parents=True, exist_ok=True)
+
+    # Mapping of destination files -> the symlink target (e.g. "../foo")
+    path_mapping: Dict[Path, Tuple[str, str]] = {}
+    # Set of directories that need to be created in the link tree before
+    # symlinking
+    dirs_to_create: Set[Path] = set()
+    # Set of __init__.py files that need to be created at the end of the
+    # link tree building if they don't exist, so that python recognizes them
+    # as modules.
+    init_py_paths = set()
+
+    # Link entries from manifests.
+    for manifest in args.module_manifests:
+        with open(manifest) as manifest_file:
+            for dest, src, origin in json.load(manifest_file):
+                dest = Path(dest)
+
+                # Add `__init__.py` files for all parent dirs (except the root).
+                if dest.suffix in _MODULE_SUFFIXES:
+                    package = dest.parent
+                    while package != Path("") and package not in init_py_paths:
+                        init_py_paths.add(package)
+                        package = package.parent
+
+                add_path_mapping(
+                    path_mapping,
+                    dirs_to_create,
+                    src,
+                    args.modules_dir / dest,
+                    origin=origin,
+                )
+
+    for manifest in args.resource_manifests + args.native_library_manifests:
+        with open(manifest) as manifest_file:
+            for dest, src, origin in json.load(manifest_file):
+                add_path_mapping(
+                    path_mapping,
+                    dirs_to_create,
+                    src,
+                    args.modules_dir / dest,
+                    origin=origin,
+                )
+
+    if args.native_library_srcs:
+        for src, dest in zip(args.native_library_srcs, args.native_library_dests):
+            new_dest = args.modules_dir / dest
+            add_path_mapping(path_mapping, dirs_to_create, src, new_dest)
+
+    if args.dwp_srcs:
+        for src, dest in zip(args.dwp_srcs, args.dwp_dests):
+            new_dest = args.modules_dir / dest
+            add_path_mapping(path_mapping, dirs_to_create, src, new_dest)
+
+    for d in dirs_to_create:
+        d.mkdir(parents=True, exist_ok=True)
+
+    for dest, (target, _origin) in path_mapping.items():
+        try:
+            os.symlink(target, dest)
+        except OSError:
+            if _lexists(dest):
+                if os.path.islink(dest):
+                    raise ValueError(
+                        "{} already exists, and is linked to {}. Cannot link to {}".format(
+                            dest, os.readlink(dest), target
+                        )
+                    )
+                else:
+                    raise ValueError(
+                        "{} already exists. Cannot link to {}".format(dest, target)
+                    )
+            else:
+                raise
+
+    # Fill in __init__.py for sources that were provided by the user
+    # These are filtered such that we only create this for sources specified
+    # by the user; if a .whl fortgets an __init__.py file, that's their problem
+    for init_py_dir in init_py_paths:
+        init_py_path = args.modules_dir / init_py_dir / "__init__.py"
+        # We still do this check because python insists on touching some read only
+        # files and blows up somtimes.
+        if not _lexists(init_py_path):
+            init_py_path.touch(exist_ok=True)
+
+
+def main() -> None:
+    args = parse_args()
+    create_modules_dir(args)
+
+
+if __name__ == "__main__":
+    main()

vendor/cxx/tools/buck/prelude/python/tools/make_source_db.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+Creates a Python Source DB JSON file containing both a rule's immediate sources
+and the sources of all transitive dependencies (e.g. for use with Pyre).
+
+Sources and dependencies are passed in via source manifest files, which are
+merged by this script:
+
+$ ./make_source_db.py \
+      --sources my_rule.manifest.json \
+      --dependency dep1.manifest.json \
+      --dependency dep2.manifest.json
+
+The output format of the source DB is:
+
+{
+  "sources": {
+    <source1-name>: <source1-path>,
+    <source2-name>: <source2-path>,
+    ...
+  },
+  "dependencies": {
+    <dep-source1-name>: <dep-source1-path>,
+    <dep-source2-name>: <dep-source2-path>,
+    ...
+  },
+}
+"""
+
+# pyre-unsafe
+
+import argparse
+import json
+import sys
+
+
+def _load(path):
+    with open(path) as f:
+        return json.load(f)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
+    parser.add_argument("--output", type=argparse.FileType("w"), default=sys.stdout)
+    parser.add_argument("--sources")
+    parser.add_argument("--dependency", action="append", default=[])
+    args = parser.parse_args(argv[1:])
+
+    db = {}
+
+    # Add sources.
+    sources = {}
+    if args.sources is not None:
+        for name, path, _ in _load(args.sources):
+            sources[name] = path
+    db["sources"] = sources
+
+    # Add dependencies.
+    dependencies = {}
+    for dep in args.dependency:
+        for name, path, origin in _load(dep):
+            prev = dependencies.get(name)
+            if prev is not None and prev[0] != path:
+                raise Exception(
+                    "Duplicate entries for {}: {} ({}) and {} ({})".format(
+                        name, path, origin, *prev
+                    ),
+                )
+            dependencies[name] = path, origin
+    db["dependencies"] = {n: p for n, (p, _) in dependencies.items()}
+
+    # Write db out.
+    json.dump(db, args.output, indent=2)
+
+
+sys.exit(main(sys.argv))

vendor/cxx/tools/buck/prelude/python/tools/make_source_db_no_deps.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+"""
+Creates a Python Source DB JSON file from Python manifest JSON file (e.g. for use with Pyre).
+
+Sources and dependencies are passed in via source manifest files, which are
+merged by this script:
+
+$ ./make_source_db_no_deps.py \
+      my_rule.manifest.json \
+      --output db_no_deps.json
+
+The output format of the source DB is:
+
+{
+  <source1-name>: <source1-path>,
+  <source2-name>: <source2-path>,
+  ...
+}
+"""
+
+import argparse
+import json
+import sys
+
+
+# pyre-fixme[3]: Return type must be annotated.
+# pyre-fixme[2]: Parameter must be annotated.
+def _load(path):
+    with open(path) as f:
+        return json.load(f)
+
+
+# pyre-fixme[3]: Return type must be annotated.
+# pyre-fixme[2]: Parameter must be annotated.
+def main(argv):
+    parser = argparse.ArgumentParser(fromfile_prefix_chars="@")
+    parser.add_argument("--output", type=argparse.FileType("w"), default=sys.stdout)
+    parser.add_argument("sources")
+    args = parser.parse_args(argv[1:])
+
+    sources = {name: path for name, path, _ in _load(args.sources)}
+    json.dump(sources, args.output, indent=2)
+
+
+sys.exit(main(sys.argv))

vendor/cxx/tools/buck/prelude/python/tools/run_inplace.py.in

@@ -0,0 +1,245 @@
+#!<PYTHON> <PYTHON_INTERPRETER_FLAGS>
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+main_module = "<MAIN_MODULE>"
+modules_dir = "<MODULES_DIR>"
+native_libs_env_var = "<NATIVE_LIBS_ENV_VAR>"
+native_libs_dir = <NATIVE_LIBS_DIR>
+native_libs_preload_env_var = "<NATIVE_LIBS_PRELOAD_ENV_VAR>"
+native_libs_preload = <NATIVE_LIBS_PRELOAD>
+interpreter_flags = "<PYTHON_INTERPRETER_FLAGS>"
+
+import os
+import platform
+import signal
+import subprocess
+import sys
+
+
+def try_resolve_possible_symlink(path):
+    import ctypes
+    import ctypes.wintypes
+
+    wintypes = ctypes.wintypes
+
+    OPEN_EXISTING = 3
+    FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+    INVALID_HANDLE_VALUE = -1
+
+    CreateFile = ctypes.windll.kernel32.CreateFileW
+    CreateFile.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD,
+                           wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD,
+                           wintypes.HANDLE]
+    CreateFile.restype = wintypes.HANDLE
+
+    CloseHandle = ctypes.windll.kernel32.CloseHandle
+    CloseHandle.argtypes = [wintypes.HANDLE]
+    CloseHandle.restype = wintypes.BOOL
+
+    GetFinalPathNameByHandle = ctypes.windll.kernel32.GetFinalPathNameByHandleW
+    GetFinalPathNameByHandle.argtypes = [wintypes.HANDLE, wintypes.LPWSTR, wintypes.DWORD, wintypes.DWORD]
+    GetFinalPathNameByHandle.restype = wintypes.DWORD
+
+    handle = INVALID_HANDLE_VALUE
+    try:
+        handle = CreateFile(path, 0, 0, None, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, None)
+        if handle == INVALID_HANDLE_VALUE:
+            return path
+
+        path_buf = ctypes.create_unicode_buffer(wintypes.MAX_PATH + 1)
+        if GetFinalPathNameByHandle(handle, path_buf, wintypes.MAX_PATH + 1, 0) == 0:
+            return path
+
+        # avoid literal backslash (ASCII octal 134) to get out of multilevel quoting hell
+        if path_buf.value.startswith(chr(0o134) + chr(0o134) + '?' + chr(0o134)):
+            return path_buf.value[4:]
+
+        return path_buf.value
+    finally:
+        if handle != INVALID_HANDLE_VALUE:
+            CloseHandle(handle)
+
+dirpath = os.path.dirname(os.path.realpath(__file__))
+if platform.system() == "Windows":
+    # Hah hah just kidding. __file__ will point to the symlink and not the
+    # actual pex we want to execute, if we're in a symlink. os.path.realpath
+    # does *not* dereference symlinks on windows until, like, 3.8 maybe.
+    dirpath = os.path.dirname(try_resolve_possible_symlink(sys.argv[0]))
+
+env_vals_to_restore = {}
+# Update the environment variable for the dynamic loader to the native
+# libraries location.
+if native_libs_dir is not None:
+    old_native_libs_dir = os.environ.get(native_libs_env_var)
+    os.environ[native_libs_env_var] = os.path.join(dirpath, native_libs_dir)
+    env_vals_to_restore[native_libs_env_var] = old_native_libs_dir
+
+# Update the environment variable for the dynamic loader to find libraries
+# to preload.
+if native_libs_preload is not None:
+    old_native_libs_preload = os.environ.get(native_libs_preload_env_var)
+    env_vals_to_restore[native_libs_preload_env_var] = old_native_libs_preload
+
+    # On macos, preloaded libs are found via paths.
+    os.environ[native_libs_preload_env_var] = ":".join(
+        os.path.join(dirpath, native_libs_dir, l)
+        for l in native_libs_preload.split(":")
+    )
+
+# Allow users to decorate the main module. In normal Python invocations this
+# can be done by prefixing the arguments with `-m decoratingmodule`. It's not
+# that easy for par files. The startup script below sets up `sys.path` from
+# within the Python interpreter. Enable decorating the main module after
+# `sys.path` has been setup by setting the PAR_MAIN_OVERRIDE environment
+# variable.
+decorate_main_module = os.environ.pop("PAR_MAIN_OVERRIDE", None)
+if decorate_main_module:
+    # Pass the original main module as environment variable for the process.
+    # Allowing the decorating module to pick it up.
+    os.environ["PAR_MAIN_ORIGINAL"] = main_module
+    main_module = decorate_main_module
+
+module_call = "runpy._run_module_as_main({main_module!r}, False)".format(
+    main_module=main_module
+)
+
+# Allow users to run the main module under pdb. Encode the call into the
+# startup script, because pdb does not support the -c argument we use to invoke
+# our startup wrapper.
+#
+# Note: use pop to avoid leaking the environment variable to the child process.
+if os.environ.pop("PYTHONDEBUGWITHPDB", None):
+    # Support passing initial commands to pdb. We cannot pass the -c argument
+    # to pdb. Instead, allow users to pass initial commands through the
+    # PYTHONPDBINITIALCOMMANDS env var, separated by the | character.
+    initial_commands = []
+    if "PYTHONPDBINITIALCOMMANDS" in os.environ:
+        # Note: use pop to avoid leaking the environment variable to the child
+        # process.
+        initial_commands_string = os.environ.pop("PYTHONPDBINITIALCOMMANDS", None)
+        initial_commands = initial_commands_string.split("|")
+
+    # Note: indentation of this block of code is important as it gets included
+    # in the bigger block below.
+    module_call = """
+    from pdb import Pdb
+    pdb = Pdb()
+    pdb.rcLines.extend({initial_commands!r})
+    pdb.runcall(runpy._run_module_as_main, {main_module!r}, False)
+""".format(
+        main_module=main_module,
+        initial_commands=initial_commands,
+    )
+
+# Note: this full block of code will be included as the argument to Python,
+# and will be the first thing that shows up in the process arguments as displayed
+# by programs like ps and top.
+#
+# We include arg0 at the start of this comment just to make it more visible what program
+# is being run in the ps and top output.
+STARTUP = """\
+# {arg0!r}
+# Wrap everything in a private function to prevent globals being captured by
+# the `runpy._run_module_as_main` below.
+def __run():
+    import sys
+
+    # We set the paths beforehand to have a minimal amount of imports before
+    # nuking PWD from sys.path. Otherwise, there can be problems if someone runs
+    # from a directory with a similarly named file, even if their code is properly
+    # namespaced. e.g. if one has foo/bar/contextlib.py and while in foo/bar runs
+    # `buck run foo/bar:bin`, runpy will fail as it tries to import
+    # foo/bar/contextlib.py. You're just out of luck if you have sys.py or os.py
+
+    # Set `argv[0]` to the executing script.
+    assert sys.argv[0] == '-c'
+    sys.argv[0] = {arg0!r}
+
+    # Replace the working directory with location of the modules directory.
+    assert sys.path[0] == ''
+    sys.path[0] = {pythonpath!r}
+
+    import os
+    import runpy
+
+    def setenv(var, val):
+        if val is None:
+            os.environ.pop(var, None)
+        else:
+            os.environ[var] = val
+
+    def restoreenv(d):
+        for k, v in d.items():
+            setenv(k, v)
+
+    restoreenv({env_vals!r})
+    {module_call}
+
+__run()
+""".format(
+    arg0=sys.argv[0],
+    pythonpath=os.path.join(dirpath, modules_dir),
+    env_vals=env_vals_to_restore,
+    main_module=main_module,
+    this_file=__file__,
+    module_call=module_call,
+)
+
+args = [sys.executable]
+if interpreter_flags:
+    args.append(interpreter_flags)
+args.extend(["-c", STARTUP])
+
+# Default to 'd' warnings, but allow users to control this via PYTHONWARNINGS
+# The -E causes python to ignore all PYTHON* environment vars so we have to
+# pass this down using the command line.
+warnings = os.environ.get("PYTHONWARNINGS", "d").split(",")
+for item in reversed(warnings):
+    args.insert(1, "-W{0}".format(item.strip()))
+
+# Allow users to disable byte code generation by setting the standard environment var.
+# Same as above, because of -E we have to pass this down using the command line.
+if "PYTHONDONTWRITEBYTECODE" in os.environ:
+    args.insert(1, "-B")
+
+# Python 3.7 allows benchmarking import time with this variable. Similar issues to
+# PYTHONDONTWRITEBYTECODE above. If using an earlier version of python... dont set this
+# Make sure we only run this on cpython where it's supported (python2 will fail
+# if given an unknown -X)
+if (
+    "PYTHONPROFILEIMPORTTIME" in os.environ
+    and platform.python_implementation() == "CPython"
+    and (sys.version_info[0], sys.version_info[1]) >= (3, 7)
+):
+    args[1:1] = ["-X", "importtime"]
+
+if platform.system() == "Windows":
+    # exec on Windows is not true exec - there is only 'spawn' ('CreateProcess').
+    # However, creating processes unnecessarily is painful, so we only do the spawn
+    # path if we have to, which is on Windows. That said, this complicates signal
+    # handling, so we need to set up some signal forwarding logic.
+
+    p = subprocess.Popen(args + sys.argv[1:])
+
+    def handler(signum, frame):
+        # If we're getting this, we need to forward signum to subprocesses
+        if signum == signal.SIGINT:
+            p.send_signal(signal.CTRL_C_EVENT)
+        elif signum == signal.SIGBREAK:
+            p.send_signal(signal.CTRL_BREAK_EVENT)
+        else:
+            # shouldn't happen, we should be killed instead
+            p.terminate()
+
+    signal.signal(signal.SIGINT, handler)
+    signal.signal(signal.SIGBREAK, handler)
+
+    p.wait()
+    sys.exit(p.returncode)
+else:
+    os.execv(sys.executable, args + sys.argv[1:])

vendor/cxx/tools/buck/prelude/python/tools/run_inplace_lite.py.in

@@ -0,0 +1,85 @@
+#!<PYTHON> <PYTHON_INTERPRETER_FLAGS>
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under both the MIT license found in the
+# LICENSE-MIT file in the root directory of this source tree and the Apache
+# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
+# of this source tree.
+
+main_module = "<MAIN_MODULE>"
+modules_dir = "<MODULES_DIR>"
+
+# Wrap everything in a private function to prevent globals being captured by
+# the `runpy._run_module_as_main` below.
+def __run():
+    import sys
+    import os
+    import platform
+
+    def try_resolve_possible_symlink(path):
+        import ctypes
+        import ctypes.wintypes
+
+        wintypes = ctypes.wintypes
+
+        OPEN_EXISTING = 3
+        FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+        INVALID_HANDLE_VALUE = -1
+
+        CreateFile = ctypes.windll.kernel32.CreateFileW
+        CreateFile.argtypes = [wintypes.LPCWSTR, wintypes.DWORD, wintypes.DWORD,
+                               wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD,
+                               wintypes.HANDLE]
+        CreateFile.restype = wintypes.HANDLE
+
+        CloseHandle = ctypes.windll.kernel32.CloseHandle
+        CloseHandle.argtypes = [wintypes.HANDLE]
+        CloseHandle.restype = wintypes.BOOL
+
+        GetFinalPathNameByHandle = ctypes.windll.kernel32.GetFinalPathNameByHandleW
+        GetFinalPathNameByHandle.argtypes = [wintypes.HANDLE, wintypes.LPWSTR, wintypes.DWORD, wintypes.DWORD]
+        GetFinalPathNameByHandle.restype = wintypes.DWORD
+
+        handle = INVALID_HANDLE_VALUE
+        try:
+            handle = CreateFile(path, 0, 0, None, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, None)
+            if handle == INVALID_HANDLE_VALUE:
+                return path
+
+            path_buf = ctypes.create_unicode_buffer(wintypes.MAX_PATH + 1)
+            if GetFinalPathNameByHandle(handle, path_buf, wintypes.MAX_PATH + 1, 0) == 0:
+                return path
+
+            # avoid literal backslash (ASCII octal 134) to get out of multilevel quoting hell
+            if path_buf.value.startswith(chr(0o134) + chr(0o134) + '?' + chr(0o134)):
+                return path_buf.value[4:]
+
+            return path_buf.value
+        finally:
+            if handle != INVALID_HANDLE_VALUE:
+                CloseHandle(handle)
+
+    # We set the paths beforehand to have a minimal amount of imports before
+    # nuking PWD from sys.path. Otherwise, there can be problems if someone runs
+    # from a directory with a similarly named file, even if their code is properly
+    # namespaced. e.g. if one has foo/bar/contextlib.py and while in foo/bar runs
+    # `buck run foo/bar:bin`, runpy will fail as it tries to import
+    # foo/bar/contextlib.py. You're just out of luck if you have sys.py or os.py
+
+    dirpath = os.path.dirname(os.path.realpath(__file__))
+    if platform.system() == "Windows":
+        # Hah hah just kidding. __file__ will point to the symlink and not the
+        # actual pex we want to execute, if we're in a symlink. os.path.realpath
+        # does *not* dereference symlinks on windows until, like, 3.8 maybe.
+        dirpath = os.path.dirname(try_resolve_possible_symlink(sys.argv[0]))
+
+    # Replace the working directory with location of the modules directory.
+    sys.path[0] = os.path.join(dirpath, modules_dir)
+
+    import os
+    import runpy
+
+    runpy._run_module_as_main(main_module, False)
+
+
+__run()