# Copyright (c) Meta Platforms, Inc. and affiliates. # # This source code is licensed under both the MIT license found in the # LICENSE-MIT file in the root directory of this source tree and the Apache # License, Version 2.0 found in the LICENSE-APACHE file in the root directory # of this source tree. load("@prelude//cxx:compile.bzl", "CxxSrcWithFlags") load("@prelude//cxx:cxx.bzl", "get_cxx_auto_link_group_specs") load("@prelude//cxx:cxx_executable.bzl", "cxx_executable") load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo") load( "@prelude//cxx:cxx_types.bzl", "CxxRuleConstructorParams", ) load("@prelude//cxx:headers.bzl", "cxx_get_regular_cxx_headers_layout") load("@prelude//cxx:link_groups.bzl", "get_link_group_info") load( "@prelude//cxx:omnibus.bzl", "all_deps", "create_omnibus_libraries", "get_excluded", "get_omnibus_graph", "get_roots", ) load( "@prelude//cxx:preprocessor.bzl", "CPreprocessor", "cxx_inherited_preprocessor_infos", ) load( "@prelude//linking:link_info.bzl", "Linkage", ) load( "@prelude//linking:linkable_graph.bzl", "create_linkable_graph", ) load( "@prelude//linking:linkables.bzl", "linkables", ) load( "@prelude//utils:types.bzl", "unchecked", # @unused Used as a type ) load("@prelude//utils:utils.bzl", "expect", "flatten", "value_or") load("@prelude//paths.bzl", "paths") load("@prelude//resources.bzl", "gather_resources") load(":compile.bzl", "compile_manifests") load( ":interface.bzl", "PythonLibraryInterface", # @unused Used as a type ) load(":make_pex.bzl", "PexModules", "PexProviders", "make_pex") load( ":manifest.bzl", "create_manifest_for_extensions", "create_manifest_for_source_dir", "create_manifest_for_source_map", ) load(":native_python_util.bzl", "merge_cxx_extension_info") load(":python.bzl", "info_to_interface") load( ":python_library.bzl", "create_python_library_info", "gather_dep_libraries", "py_resources", "qualify_srcs", ) load(":source_db.bzl", "create_source_db", "create_source_db_no_deps") load(":toolchain.bzl", "NativeLinkStrategy", "PackageStyle", "PythonPlatformInfo", "PythonToolchainInfo", "get_platform_attr") OmnibusMetadataInfo = provider(fields = ["omnibus_libs", "omnibus_graph"]) def _link_strategy(ctx: "context") -> NativeLinkStrategy.type: if ctx.attrs.native_link_strategy != None: return NativeLinkStrategy(ctx.attrs.native_link_strategy) return NativeLinkStrategy(ctx.attrs._python_toolchain[PythonToolchainInfo].native_link_strategy) def _package_style(ctx: "context") -> PackageStyle.type: if ctx.attrs.package_style != None: return PackageStyle(ctx.attrs.package_style.lower()) return PackageStyle(ctx.attrs._python_toolchain[PythonToolchainInfo].package_style) # We do a lot of merging extensions, so don't use O(n) type annotations def _merge_extensions( extensions: unchecked({str.type: ("_a", "label")}), incoming_label: unchecked("label"), incoming_extensions: unchecked({str.type: "_a"})) -> None: """ Merges a incoming_extensions into `extensions`. Fails if duplicate dests exist. """ for extension_name, incoming_artifact in incoming_extensions.items(): existing = extensions.get(extension_name) if existing != None and existing[0] != incoming_artifact: existing_artifact, existing_label = existing error = ( "Duplicate extension: {}! Conflicting mappings:\n" + "{} from {}\n" + "{} from {}" ) fail( error.format( extension_name, existing_artifact, existing_label, incoming_artifact, incoming_label, ), ) extensions[extension_name] = (incoming_artifact, incoming_label) def python_executable( ctx: "context", main_module: str.type, srcs: {str.type: "artifact"}, resources: {str.type: ("artifact", ["_arglike"])}, compile: bool.type = False) -> PexProviders.type: # Returns a three tuple: the Python binary, all its potential runtime files, # and a provider for its source DB. # TODO(nmj): See if people are actually setting cxx_platform here. Really # feels like it should be a property of the python platform python_platform = ctx.attrs._python_toolchain[PythonPlatformInfo] cxx_platform = ctx.attrs._cxx_toolchain[CxxPlatformInfo] raw_deps = ( [ctx.attrs.deps] + get_platform_attr(python_platform, cxx_platform, ctx.attrs.platform_deps) ) # `preload_deps` is used later to configure `LD_PRELOAD` environment variable, # here we make the actual libraries to appear in the distribution. # TODO: make fully consistent with its usage later raw_deps.append(ctx.attrs.preload_deps) python_deps, shared_deps = gather_dep_libraries(raw_deps) src_manifest = None bytecode_manifest = None if srcs: src_manifest = create_manifest_for_source_map(ctx, "srcs", srcs) bytecode_manifest = create_manifest_for_source_dir( ctx, "bytecode", compile_manifests(ctx, [src_manifest]), ) all_resources = {} all_resources.update(resources) for cxx_resources in gather_resources(ctx.label, deps = flatten(raw_deps)).values(): for name, resource in cxx_resources.items(): all_resources[paths.join("__cxx_resources__", name)] = resource library_info = create_python_library_info( ctx.actions, ctx.label, srcs = src_manifest, resources = py_resources(ctx, all_resources) if all_resources else None, bytecode = bytecode_manifest, deps = python_deps, shared_libraries = shared_deps, ) source_db = create_source_db(ctx, src_manifest, python_deps) source_db_no_deps = create_source_db_no_deps(ctx, srcs) exe = convert_python_library_to_executable( ctx, main_module, info_to_interface(library_info), flatten(raw_deps), compile, ) exe.sub_targets.update({ "source-db": [source_db], "source-db-no-deps": [source_db_no_deps, library_info], }) return exe def convert_python_library_to_executable( ctx: "context", main_module: "string", library: PythonLibraryInterface.type, deps: ["dependency"], compile: bool.type = False) -> PexProviders.type: extra = {} python_toolchain = ctx.attrs._python_toolchain[PythonToolchainInfo] package_style = _package_style(ctx) # Convert preloaded deps to a set of their names to be loaded by. preload_labels = {d.label: None for d in ctx.attrs.preload_deps} preload_names = { name: None for name, shared_lib in library.shared_libraries().items() if shared_lib.label in preload_labels } extensions = {} extra_manifests = None for manifest in library.iter_manifests(): if manifest.extensions: _merge_extensions(extensions, manifest.label, manifest.extensions) # If we're using omnibus linking, re-link libraries and extensions and # update the libraries we'll pull into the final binary. if _link_strategy(ctx) == NativeLinkStrategy("merged"): # Collect omnibus info from deps. linkable_graph = create_linkable_graph( ctx, deps = deps, ) omnibus_graph = get_omnibus_graph( graph = linkable_graph, # Add in any potential native root targets from our first-order deps. roots = get_roots(ctx.label, deps), # Exclude preloaded deps from omnibus linking, to prevent preloading # the monolithic omnibus library. excluded = get_excluded(deps = ctx.attrs.preload_deps), ) # Link omnibus libraries. omnibus_libs = create_omnibus_libraries( ctx, omnibus_graph, ctx.attrs.linker_flags, prefer_stripped_objects = ctx.attrs.prefer_stripped_native_objects, ) # Extract re-linked extensions. extensions = { dest: (omnibus_libs.roots[label].product.shared_library, label) for dest, (_, label) in extensions.items() } native_libs = omnibus_libs.libraries if python_toolchain.emit_omnibus_metadata: omnibus_linked_obj = omnibus_libs.omnibus omnibus_info = DefaultInfo() if omnibus_linked_obj: omnibus_info = DefaultInfo( default_outputs = [omnibus_linked_obj.output], sub_targets = { "dwp": [DefaultInfo(default_outputs = [omnibus_linked_obj.dwp] if omnibus_linked_obj.dwp else [])], }, ) extra["omnibus"] = [ omnibus_info, OmnibusMetadataInfo( omnibus_libs = omnibus_libs, omnibus_graph = omnibus_graph, ), ] exclusion_roots = ctx.actions.write_json("omnibus/exclusion_roots.json", omnibus_libs.exclusion_roots) extra["omnibus-exclusion-roots"] = [DefaultInfo(default_outputs = [exclusion_roots])] roots = ctx.actions.write_json("omnibus/roots.json", omnibus_libs.roots) extra["omnibus-roots"] = [DefaultInfo(default_outputs = [roots])] omnibus_excluded = ctx.actions.write_json("omnibus/excluded.json", omnibus_libs.excluded) extra["omnibus-excluded"] = [DefaultInfo(default_outputs = [omnibus_excluded])] omnibus_graph_json = ctx.actions.write_json("omnibus_graph.json", omnibus_graph) extra["linkable-graph"] = [DefaultInfo(default_outputs = [omnibus_graph_json])] elif _link_strategy(ctx) == NativeLinkStrategy("native"): expect(package_style == PackageStyle("standalone"), "native_link_strategy=native is only supported for standalone builds") executable_deps = ctx.attrs.executable_deps extension_info = merge_cxx_extension_info(ctx.actions, deps + executable_deps) inherited_preprocessor_info = cxx_inherited_preprocessor_infos(executable_deps) # Generate an additional C file as input static_extension_info_out = ctx.actions.declare_output("static_extension_info.cpp") cmd = cmd_args(python_toolchain.generate_static_extension_info[RunInfo]) cmd.add(cmd_args(static_extension_info_out.as_output(), format = "--output={}")) cmd.add(cmd_args(["{}:{}".format(k, v) for k, v in extension_info.python_module_names.items()], format = "--extension={}")) # TODO we don't need to do this ... ctx.actions.run(cmd, category = "generate_static_extension_info") extra["static_extension_info"] = [DefaultInfo(default_outputs = [static_extension_info_out])] cxx_executable_srcs = [ CxxSrcWithFlags(file = ctx.attrs.cxx_main, flags = []), CxxSrcWithFlags(file = static_extension_info_out, flags = []), ] extra_preprocessors = [] if ctx.attrs.par_style == "native": extra_preprocessors.append(CPreprocessor(args = ["-DNATIVE_PAR_STYLE=1"])) # All deps inolved in the link. link_deps = ( linkables(executable_deps) + list(extension_info.linkable_providers.traverse()) ) link_group_info = get_link_group_info(ctx, [d.linkable_graph for d in link_deps]) impl_params = CxxRuleConstructorParams( rule_type = "python_binary", headers_layout = cxx_get_regular_cxx_headers_layout(ctx), srcs = cxx_executable_srcs, extra_link_flags = ctx.attrs.linker_flags, extra_preprocessors = extra_preprocessors, extra_preprocessors_info = inherited_preprocessor_info, extra_link_deps = link_deps, exe_shared_libs_link_tree = False, force_full_hybrid_if_capable = True, link_group_info = link_group_info, auto_link_group_specs = get_cxx_auto_link_group_specs(ctx, link_group_info), ) executable_info, _, _ = cxx_executable(ctx, impl_params) extra["native-executable"] = [DefaultInfo(default_outputs = [executable_info.binary])] linkable_graph = create_linkable_graph( ctx, deps = deps, ) # Add any shared only libs into the par nodes = linkable_graph.nodes.traverse() node_map = {} native_libs = {} shared_only = [] for node in filter(None, nodes): if node.linkable: node_map[node.label] = node.linkable if node.linkable.preferred_linkage == Linkage("shared"): shared_only.append(node.label) for label in all_deps(node_map, shared_only): for name, shared_lib in node_map[label].shared_libs.items(): native_libs[name] = shared_lib # Include shared libs from e.g. link groups. native_libs.update(executable_info.shared_libs) # Include dlopen-able shared lib deps. for libs in extension_info.shared_libraries.traverse(): for name, shared_lib in libs.libraries.items(): native_libs[name] = shared_lib.lib # Add sub-targets for libs. for name, lib in native_libs.items(): extra[name] = [DefaultInfo(default_outputs = [lib.output])] # TODO expect(len(executable_info.runtime_files) == 0, "OH NO THERE ARE RUNTIME FILES") artifacts = dict(extension_info.artifacts) artifacts["runtime/bin/{}".format(ctx.attrs.executable_name)] = executable_info.binary extra_manifests = create_manifest_for_source_map(ctx, "extension_stubs", artifacts) extensions = {} else: native_libs = {name: shared_lib.lib for name, shared_lib in library.shared_libraries().items()} # Combine sources and extensions into a map of all modules. pex_modules = PexModules( manifests = library.manifests(), extra_manifests = extra_manifests, compile = compile, extensions = create_manifest_for_extensions( ctx, extensions, dwp = ctx.attrs.package_split_dwarf_dwp, ) if extensions else None, ) # Create the map of native libraries to their artifacts and whether they # need to be preloaded. Note that we merge preload deps into regular deps # above, before gathering up all native libraries, so we're guaranteed to # have all preload libraries (and their transitive deps) here. shared_libraries = {} for name, lib in native_libs.items(): shared_libraries[name] = lib, name in preload_names hidden_resources = library.hidden_resources() if library.has_hidden_resources() else None # Build the PEX. pex = make_pex( ctx, python_toolchain, ctx.attrs.bundled_runtime, package_style, ctx.attrs.build_args, pex_modules, shared_libraries, main_module, hidden_resources, ) pex.sub_targets.update(extra) return pex def python_binary_impl(ctx: "context") -> ["provider"]: main_module = ctx.attrs.main_module if ctx.attrs.main_module != None and ctx.attrs.main != None: fail("Only one of main_module or main may be set. Prefer main_module as main is considered deprecated") elif ctx.attrs.main != None: base_module = ctx.attrs.base_module if base_module == None: base_module = ctx.label.package.replace("/", ".") if base_module != "": base_module += "." main_module = base_module + ctx.attrs.main.short_path.replace("/", ".") if main_module.endswith(".py"): main_module = main_module[:-3] srcs = {} if ctx.attrs.main != None: srcs[ctx.attrs.main.short_path] = ctx.attrs.main srcs = qualify_srcs(ctx.label, ctx.attrs.base_module, srcs) pex = python_executable( ctx, main_module, srcs, {}, compile = value_or(ctx.attrs.compile, False), ) return [ DefaultInfo( default_outputs = pex.default_outputs, other_outputs = pex.other_outputs, sub_targets = pex.sub_targets, ), RunInfo(pex.run_cmd), ]