| | |
|---|
| | | |
|---|
| | | def main(**kwargs): |
|---|
| | | print(kwargs) |
|---|
| | | |
|---|
| | | # set random seed |
|---|
| | | tables.print() |
|---|
| | | set_all_random_seed(kwargs.get("seed", 0)) |
|---|
| | | torch.backends.cudnn.enabled = kwargs.get("cudnn_enabled", torch.backends.cudnn.enabled) |
|---|
| | | torch.backends.cudnn.benchmark = kwargs.get("cudnn_benchmark", torch.backends.cudnn.benchmark) |
|---|
| | | torch.backends.cudnn.deterministic = kwargs.get("cudnn_deterministic", True) |
|---|
| | | |
|---|
| | | local_rank = int(os.environ.get('LOCAL_RANK', 0)) |
|---|
| | | if local_rank == 0: |
|---|
| | | tables.print() |
|---|
| | | # Check if we are using DDP or FSDP |
|---|
| | | use_ddp = 'WORLD_SIZE' in os.environ and int(os.environ["WORLD_SIZE"]) > 1 |
|---|
| | | use_fsdp = kwargs.get("use_fsdp", None) |
|---|
| | |
|---|
| | | frontend = frontend_class(**kwargs["frontend_conf"]) |
|---|
| | | kwargs["frontend"] = frontend |
|---|
| | | kwargs["input_size"] = frontend.output_size() |
|---|
| | | |
|---|
| | | # import pdb; |
|---|
| | | # pdb.set_trace() |
|---|
| | | |
|---|
| | | |
|---|
| | | # build model |
|---|
| | | model_class = tables.model_classes.get(kwargs["model"]) |
|---|
| | | model = model_class(**kwargs, **kwargs["model_conf"], vocab_size=len(tokenizer.token_list)) |
|---|
