TL

TL implementation manual

Each TL is a dynamically loadable component that implements collectives primitives defined in common interface ucc_tl_iface_t (src/components/tl/ucc_tl.h) TLs are discovered in runtime and used by CLs (higher level layers that build collective schedules using TL backends)

Interface

TL interface can be split into groups: control and data.

typedef struct ucc_tl_iface {
    ucc_component_iface_t          super;
    ucs_config_global_list_entry_t tl_lib_config;
    ucs_config_global_list_entry_t tl_context_config;
    ucc_base_lib_iface_t           lib;
    ucc_base_context_iface_t       context;
    ucc_base_team_iface_t          team;
    ucc_base_coll_iface_t          coll;
    ucc_tl_service_coll_t          scoll;
    ucc_base_coll_alg_info_t *     alg_info[UCC_COLL_TYPE_NUM];
} ucc_tl_iface_t;

Control includes: ucc_base_lib_iface_t, ucc_base_context_iface_t, ucc_base_team_iface_t

• lib: tl library object constructor/destructor and get_lib_attr
• context: tl context object constructor/destructor and get_context_attr
• team: tl team constructor/destructor, team_create_test

Data includes: ucc_base_coll_iface_t

• coll: collective_init

Code structure

TL code resides under src/components/tl/new and usually contains: tl_new.h, tl_new.c, tl_new_lib.c, tl_new_context.c, tl_new_team.c, tl_new_coll.c/h

tl_new.h:

1. Main interface structure

typedef struct ucc_tl_new_iface {
    ucc_tl_iface_t super;
} ucc_tl_new_iface_t;
/* Extern iface should follow the pattern: ucc_tl_<tl_name> */
extern ucc_tl_new_iface_t ucc_tl_new;

The actual ucc_tl_new object is instantiated in tl_new.c using macro: UCC_TL_IFACE_DECLARE(new, NEW); 2. TL configuration (env var parameters). Configs are allowed for lib and for context (note, there 1-to-many relation between lib and ctx objects)

typedef struct ucc_tl_new_lib_config {
    ucc_tl_lib_config_t super;
    uint32_t            example_lib_param;
} ucc_tl_new_lib_config_t;

typedef struct ucc_tl_new_context_config {
    ucc_tl_context_config_t super;
    uint32_t                example_ctx_param;
} ucc_tl_new_context_config_t;

3. Lib structure:

typedef struct ucc_tl_new_lib {
    ucc_tl_lib_t            super;
    ucc_tl_new_lib_config_t cfg;
} ucc_tl_new_lib_t;
UCC_CLASS_DECLARE(ucc_tl_new_lib_t, const ucc_base_new_params_t *,
                  const ucc_base_config_t *);

4. Context structure:

typedef struct ucc_tl_new_context {
    ucc_tl_context_t            super;
    ucc_tl_new_context_config_t cfg;
} ucc_tl_new_context_t;
UCC_CLASS_DECLARE(ucc_tl_new_context_t, const ucc_base_context_params_t *,
                  const ucc_base_config_t *);

5. Team structure: typedef struct ucc_tl_new_team { ucc_tl_team_t super; } ucc_tl_new_team_t; UCC_CLASS_DECLARE(ucc_tl_new_team_t, ucc_base_context_t *, const ucc_base_team_params_t *);

tl_new.c

1. Contains declaration of iface:

UCC_TL_IFACE_DECLARE(new, NEW);

2. config tables for lib and ctx:

static ucc_config_field_t ucc_tl_new_lib_config_table[] = {
    {"", "", NULL, ucc_offsetof(ucc_tl_new_lib_config_t, super),
     UCC_CONFIG_TYPE_TABLE(ucc_tl_lib_config_table)},

    {"EXAMPLE_LIB_PARAM", "1",
     "param description",
     ucc_offsetof(ucc_tl_new_lib_config_t, example_lib_param),
     UCC_CONFIG_TYPE_UINT},
    {NULL}};

static ucc_config_field_t ucc_tl_new_context_config_table[] = {
    {"", "", NULL, ucc_offsetof(ucc_tl_new_context_config_t, super),
     UCC_CONFIG_TYPE_TABLE(ucc_tl_context_config_table)},

    {"EXAMPLE_CTX_PARAM", "1",
     "param description",
     ucc_offsetof(ucc_tl_new_ctx_config_t, example_ctx_param),
     UCC_CONFIG_TYPE_UINT},
    {NULL}};

3. may contain a constructor (see tl_ucp.c as example)

tl_new_lib.c

1. LIB constructor/destructor

UCC_CLASS_INIT_FUNC(ucc_tl_new_lib_t, const ucc_base_lib_params_t *params,
                    const ucc_base_config_t *config)
{
    const ucc_tl_new_lib_config_t *tl_new_config =
        ucc_derived_of(config, ucc_tl_new_lib_config_t);
    UCC_CLASS_CALL_SUPER_INIT(ucc_tl_lib_t, &ucc_tl_new.super,
                              &tl_new_config->super);
    memcpy(&self->cfg, tl_new_config, sizeof(*tl_new_config));
    tl_info(&self->super, "initialized lib object: %p", self);
    return UCC_OK;
}

UCC_CLASS_CLEANUP_FUNC(ucc_tl_new_lib_t)
{
    tl_info(&self->super, "finalizing lib object: %p", self);
}

UCC_CLASS_DEFINE(ucc_tl_new_lib_t, ucc_tl_lib_t);

2. ucc_tl_new_get_lib_attr Need to set supported THREAD_MODE, bitmask of supported coll_types and additional flags if required.

ucc_status_t ucc_tl_new_get_lib_attr(const ucc_base_lib_t *lib, /* NOLINT */
                                     ucc_base_lib_attr_t  *base_attr)
{
    ucc_tl_lib_attr_t *attr = ucc_derived_of(base_attr, ucc_tl_lib_attr_t);
    attr->super.attr.thread_mode = UCC_THREAD_SINGLE;
    attr->super.attr.coll_types  = UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_ALLOTALL;
    attr->super.flags            = UCC_BASE_LIB_FLAG_TEAM_ID_REQUIRED;
    return UCC_OK;
}