Index

A

ABSOLUTE (modalities.models.gpt2.gpt2_model.PositionTypes attribute) activation (modalities.models.coca.coca_model.TextDecoderConfig attribute) activation_checkpointing_modules (modalities.config.config.ActivationCheckpointedModelConfig attribute) activation_type (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) ActivationCheckpointedModelConfig (class in modalities.config.config) ActivationType (class in modalities.models.model) AdamOptimizerConfig (class in modalities.config.config) AdamWOptimizerConfig (class in modalities.config.config) add_cls_token (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) add_entity() (modalities.registry.registry.Registry method) add_subscriber() (modalities.logging_broker.message_broker.MessageBroker method) (modalities.logging_broker.message_broker.MessageBrokerIF method) add_value() (modalities.util.Aggregator method) Aggregator (class in modalities.util) anneal_strategy (modalities.config.config.OneCycleLRSchedulerConfig attribute) app_state (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) apply_activation_checkpointing_inplace() (in module modalities.training.activation_checkpointing) apply_rotary_pos_emb() (in module modalities.conversion.gpt2.modeling_gpt2) (modalities.models.gpt2.gpt2_model.RotaryTransform method)

AppState (class in modalities.checkpointing.stateful.app_state) AppStateFactory (class in modalities.checkpointing.stateful.app_state_factory) attention_config (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) attention_engine_type (modalities.nn.attention.AttentionConfig attribute) attention_implementation (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) attention_norm_config (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) AttentionConfig (class in modalities.models.gpt2.gpt2_model) (class in modalities.nn.attention) AttentionConfig.QueryKeyValueTransformConfig (class in modalities.models.gpt2.gpt2_model) AttentionConfig.QueryKeyValueTransformConfig.IdentityTransformConfig (class in modalities.models.gpt2.gpt2_model) AttentionConfig.QueryKeyValueTransformConfig.RotaryTransformConfig (class in modalities.models.gpt2.gpt2_model) AttentionEngineType (class in modalities.nn.attention) AttentionImplementation (class in modalities.models.gpt2.gpt2_model) AttentionPooling (class in modalities.models.coca.attention_pooling) attentions (modalities.models.huggingface_adapters.hf_adapter.ModalitiesModelOutput attribute) AttentionType (class in modalities.nn.attention) AutoModelForCausalLM (modalities.models.huggingface.huggingface_model.HuggingFaceModelTypes attribute) AutoModelForMaskedLM (modalities.models.huggingface.huggingface_model.HuggingFaceModelTypes attribute)

B

base_freq (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig.RotaryTransformConfig attribute) base_model_prefix (modalities.conversion.gpt2.modeling_gpt2.GPT2ForQuestionAnswering attribute) (modalities.conversion.gpt2.modeling_gpt2.GPT2PreTrainedModel attribute) base_model_tp_plan (modalities.conversion.gpt2.configuration_gpt2.GPT2Config attribute) base_momentum (modalities.config.config.OneCycleLRSchedulerConfig attribute) BaseReader (class in modalities.dataloader.large_file_lines_reader) Batch (class in modalities.batch) batch_dim (modalities.batch.DatasetBatch attribute) (modalities.batch.InferenceResultBatch attribute) BATCH_PROGRESS_UPDATE (modalities.logging_broker.messages.MessageTypes attribute) batch_sampler (modalities.config.config.LLMDataLoaderConfig attribute) batch_size (modalities.config.config.BatchSamplerConfig attribute) BatchSamplerConfig (class in modalities.config.config) BatchStateError betas (modalities.config.config.AdamOptimizerConfig attribute) (modalities.config.config.AdamWOptimizerConfig attribute) BF16 (modalities.config.config.PrecisionEnum attribute) BF_16 (modalities.running_env.env_utils.MixedPrecisionSettings attribute) (modalities.running_env.env_utils.PyTorchDtypes attribute)

BF_16_WORKING (modalities.running_env.env_utils.MixedPrecisionSettings attribute) bias (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.components.layer_norms.LayerNormConfig attribute) (modalities.models.components.layer_norms.RMSLayerNormConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) bias_attn_pool (modalities.models.coca.coca_model.CoCaConfig attribute) biases (modalities.nn.model_initialization.parameter_name_filters.RegexFilter attribute) block_names (modalities.config.config.CompiledModelConfig attribute) (modalities.config.config.FSDP1CheckpointLoadingConfig attribute) (modalities.config.config.FSDP2WrappedModelConfig attribute) (modalities.config.config.FSDPWrappedModelConfig attribute) block_size (modalities.config.config.PackedMemMapDatasetMegatronConfig attribute) (modalities.models.coca.coca_model.TextDecoderConfig attribute) build_components() (modalities.config.component_factory.ComponentFactory method) build_hf_tokenization_components() (in module modalities.utils.verify_tokenization_consistency) build_sp_tokenization_components() (in module modalities.utils.verify_tokenization_consistency) BY_REFERENCE (modalities.config.config.PassType attribute) BY_VALUE (modalities.config.config.PassType attribute)

C

calculate_hashed_seed() (in module modalities.utils.seeding) CAUSAL_SELF_ATTENTION (modalities.nn.attention.AttentionType attribute) CausalSelfAttention (class in modalities.models.gpt2.gpt2_model) check_converted_model() (in module modalities.conversion.gpt2.conversion_model) check_divisibility() (modalities.models.gpt2.gpt2_model.GPT2LLMConfig method) check_std_and_hidden_dim() (modalities.nn.model_initialization.initialization_routines.PlainInitializationConfig method) check_totals_steps_and_epchs() (modalities.config.config.OneCycleLRSchedulerConfig method) checkpoint_dir_path (modalities.config.config.DCPAppStateConfig attribute) checkpoint_folder_path (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.DCPWarmstartCheckpointPaths attribute) CHECKPOINT_FOLDER_STRUCTURE (modalities.checkpointing.fsdp.fsdp_checkpoint_saving.DCPCheckpointSaving attribute) checkpoint_loading (modalities.config.config.FSDP1CheckpointedModelConfig attribute) (modalities.config.config.FSDP1CheckpointedOptimizerConfig attribute) checkpoint_path (modalities.config.config.DCPCheckpointSavingConfig attribute) (modalities.config.config.FSDP1CheckpointedModelConfig attribute) (modalities.config.config.FSDP1CheckpointedOptimizerConfig attribute) (modalities.config.config.FSDP1CheckpointSavingConfig attribute) (modalities.utils.number_conversion.NumberConversionFromCheckpointPathConfig attribute) checkpoint_saving (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) checkpoint_saving_execution (modalities.config.config.CheckpointSavingConfig attribute) checkpoint_saving_path (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.Paths attribute) checkpoint_saving_strategy (modalities.config.config.CheckpointSavingConfig attribute) CHECKPOINT_STRUCTURE (modalities.checkpointing.fsdp.fsdp_checkpoint_saving.FSDP1CheckpointSaving attribute) CheckpointConversion (class in modalities.checkpointing.checkpoint_conversion) CHECKPOINTED_MODEL (modalities.models.utils.ModelTypeEnum attribute) checkpointing_interval_in_steps (modalities.config.instantiation_models.Intervals attribute) CheckpointingEntityType (class in modalities.checkpointing.fsdp.fsdp_checkpoint_saving) CheckpointingError CheckpointingInstruction (class in modalities.checkpointing.checkpoint_saving_instruction) checkpoints_to_delete (modalities.checkpointing.checkpoint_saving_instruction.CheckpointingInstruction attribute) CheckpointSaving (class in modalities.checkpointing.checkpoint_saving) CheckpointSavingConfig (class in modalities.config.config) CheckpointSavingExecutionABC (class in modalities.checkpointing.checkpoint_saving_execution) CheckpointSavingStrategyIF (class in modalities.checkpointing.checkpoint_saving_strategies) Chunking (class in modalities.preprocessing.create_chunks) clip_grad_norm_() (modalities.training.gradient_clipping.fsdp_gradient_clipper.FSDP2GradientClipper static method) clip_gradients() (modalities.training.gradient_clipping.fsdp_gradient_clipper.DummyGradientClipper method) (modalities.training.gradient_clipping.fsdp_gradient_clipper.FSDP1GradientClipper method) (modalities.training.gradient_clipping.fsdp_gradient_clipper.FSDP1LoggingOnlyGradientClipper method) (modalities.training.gradient_clipping.fsdp_gradient_clipper.FSDP2GradientClipper method) (modalities.training.gradient_clipping.fsdp_gradient_clipper.FSDP2LoggingOnlyGradientClipper method) (modalities.training.gradient_clipping.gradient_clipper.GradientClipperIF method) CLMCrossEntropyLoss (class in modalities.loss_functions) CLMCrossEntropyLossConfig (class in modalities.config.config) close() (modalities.dataloader.large_file_lines_reader.LargeFileLinesReader method) CoCa (class in modalities.models.coca.coca_model) COCA (modalities.nn.model_initialization.parameter_name_filters.SupportWeightInitModels attribute) CoCaCollateFnConfig (class in modalities.models.coca.collator) CoCaCollatorFn (class in modalities.models.coca.collator) CoCaConfig (class in modalities.models.coca.coca_model) collate_fn (modalities.config.config.LLMDataLoaderConfig attribute) CollateFnIF (class in modalities.models.gpt2.collator) CombinedDataset (class in modalities.dataloader.dataset) CombinedDatasetConfig (class in modalities.config.config)

CompiledModelConfig (class in modalities.config.config) component_config_type (modalities.registry.components.ComponentEntity attribute) component_key (modalities.registry.components.ComponentEntity attribute) component_type (modalities.registry.components.ComponentEntity attribute) ComponentEntity (class in modalities.registry.components) ComponentFactory (class in modalities.config.component_factory) ComposedInitializationRoutines (class in modalities.nn.model_initialization.composed_initialization) ComposedModelInitializationConfig (class in modalities.nn.model_initialization.composed_initialization) compute() (modalities.utils.mfu.GPT2MFUCalculator method) (modalities.utils.mfu.MFUCalculatorABC method) config (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig attribute) (modalities.models.gpt2.gpt2_model.LayerNormWrapperConfig attribute) config_class (modalities.conversion.gpt2.modeling_gpt2.GPT2PreTrainedModel attribute) (modalities.models.huggingface_adapters.hf_adapter.HFModelAdapter attribute) config_file_path (modalities.config.config.WandBEvaluationResultSubscriberConfig attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) ConfigError consistency_enforcement (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) ConsistencyEnforcement (class in modalities.config.instantiation_models) ConstantLRSchedulerConfig (class in modalities.config.config) consume_dict() (modalities.logging_broker.subscriber.MessageSubscriberIF method) (modalities.logging_broker.subscriber_impl.progress_subscriber.DummyProgressSubscriber method) (modalities.logging_broker.subscriber_impl.progress_subscriber.RichProgressSubscriber method) (modalities.logging_broker.subscriber_impl.results_subscriber.DummyResultSubscriber method) (modalities.logging_broker.subscriber_impl.results_subscriber.RichResultSubscriber method) (modalities.logging_broker.subscriber_impl.results_subscriber.WandBEvaluationResultSubscriber method) consume_message() (modalities.logging_broker.subscriber.MessageSubscriberIF method) (modalities.logging_broker.subscriber_impl.progress_subscriber.DummyProgressSubscriber method) (modalities.logging_broker.subscriber_impl.progress_subscriber.RichProgressSubscriber method) (modalities.logging_broker.subscriber_impl.results_subscriber.DummyResultSubscriber method) (modalities.logging_broker.subscriber_impl.results_subscriber.RichResultSubscriber method) (modalities.logging_broker.subscriber_impl.results_subscriber.WandBEvaluationResultSubscriber method) context_parallel_degree (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) convert_base_model_config_to_dict() (in module modalities.config.utils) convert_gpt2() (in module modalities.conversion.gpt2.convert_gpt2) convert_model_checkpoint() (in module modalities.conversion.gpt2.conversion_model) convert_model_config() (in module modalities.conversion.gpt2.conversion_model) convert_pytorch_to_hf_checkpoint() (in module modalities.api) (modalities.checkpointing.checkpoint_conversion.CheckpointConversion method) convert_tokenizer() (in module modalities.conversion.gpt2.conversion_tokenizer) CosineAnnealingLRSchedulerConfig (class in modalities.config.config) CP (modalities.running_env.fsdp.device_mesh.ParallelismDegrees attribute) create_filtered_tokenized_dataset() (in module modalities.api) create_index() (modalities.dataloader.create_index.IndexGenerator method) create_raw_data_index() (in module modalities.api) create_shuffled_dataset_chunk() (in module modalities.api) create_shuffled_jsonl_dataset_chunk() (in module modalities.api) CROSS_ATTENTION (modalities.nn.attention.AttentionType attribute) cuda_env (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) CudaEnv (class in modalities.running_env.cuda_env) CudaEnvSettings (class in modalities.config.instantiation_models) cumulative_seqlens_k (modalities.conversion.gpt2.modeling_gpt2.KwargsForCausalLM attribute) cumulative_seqlens_q (modalities.conversion.gpt2.modeling_gpt2.KwargsForCausalLM attribute) cycle_momentum (modalities.config.config.OneCycleLRSchedulerConfig attribute)

D

DAO_FLASH (modalities.models.gpt2.gpt2_model.AttentionImplementation attribute) data (modalities.dataloader.create_packed_data.EmbeddedStreamData property) data_parallel_replicate_degree (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) data_parallel_shard_degree (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) DATA_SECTION_LENGTH_IN_BYTES (modalities.dataloader.create_packed_data.EmbeddedStreamData attribute) (modalities.dataloader.dataset.PackedMemMapDatasetBase attribute) data_source (modalities.config.config.SequentialSamplerConfig attribute) dataloader_tag (modalities.batch.EvaluationResultBatch attribute) (modalities.config.config.LLMDataLoaderConfig attribute) (modalities.dataloader.dataloader.LLMDataLoader property) (modalities.logging_broker.messages.ProgressUpdate attribute) DataloaderFactory (class in modalities.dataloader.dataloader_factory) Dataset (class in modalities.dataloader.dataset) dataset (modalities.config.config.DistributedSamplerConfig attribute) (modalities.config.config.LLMDataLoaderConfig attribute) (modalities.config.config.ResumableDistributedSamplerConfig attribute) dataset_path (modalities.utils.number_conversion.NumTokensFromPackedMemMapDatasetContinuousConfig attribute) DatasetBatch (class in modalities.batch) DatasetFactory (class in modalities.dataloader.dataset_factory) DatasetNotFoundError datasets (modalities.config.config.CombinedDatasetConfig attribute) DataShuffler (class in modalities.preprocessing.shuffle_data) DCPAppStateConfig (class in modalities.config.config) DCPCheckpointLoading (class in modalities.checkpointing.fsdp.fsdp_checkpoint_loading) DCPCheckpointLoadingConfig (class in modalities.config.config) DCPCheckpointSaving (class in modalities.checkpointing.fsdp.fsdp_checkpoint_saving) DCPCheckpointSavingConfig (class in modalities.config.config) debug (modalities.config.config.CompiledModelConfig attribute) decimal_places (modalities.batch.ResultItem attribute) decode() (modalities.tokenization.tokenizer_wrapper.PreTrainedHFTokenizer method) (modalities.tokenization.tokenizer_wrapper.PreTrainedSPTokenizer method) (modalities.tokenization.tokenizer_wrapper.TokenizerWrapper method) DEFAULT_ATTENTION (modalities.nn.attention.AttentionEngineType attribute) default_index_path() (modalities.dataloader.large_file_lines_reader.LargeFileLinesReader static method) detach() (modalities.batch.DatasetBatch method) (modalities.batch.InferenceResultBatch method) (modalities.batch.TorchDeviceMixin method)

device (modalities.batch.DatasetBatch property) (modalities.batch.InferenceResultBatch property) (modalities.batch.TorchDeviceMixin property) (modalities.config.config.TorchCheckpointLoadingConfig attribute) (modalities.config.instantiation_models.TextGenerationInstantiationModel.TextGenerationSettings attribute) (modalities.inference.text.config.TextInferenceComponentConfig attribute) device_mesh (modalities.config.config.FSDP2WrappedModelConfig attribute) device_type (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) DeviceMeshConfig (class in modalities.running_env.fsdp.device_mesh) directory (modalities.config.config.WandBEvaluationResultSubscriberConfig attribute) DISABLED (modalities.config.config.WandbMode attribute) distribute_message() (modalities.logging_broker.message_broker.MessageBroker method) (modalities.logging_broker.message_broker.MessageBrokerIF method) DistributedCheckpointLoadingIF (class in modalities.checkpointing.checkpoint_loading) DistributedSamplerConfig (class in modalities.config.config) div_factor (modalities.config.config.OneCycleLRSchedulerConfig attribute) DP_REPLICATE (modalities.running_env.fsdp.device_mesh.ParallelismDegrees attribute) DP_SHARD (modalities.running_env.fsdp.device_mesh.ParallelismDegrees attribute) drop_last (modalities.config.config.BatchSamplerConfig attribute) (modalities.config.config.DistributedSamplerConfig attribute) (modalities.config.config.ResumableDistributedSamplerConfig attribute) dropout (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) dst_path (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) DummyDataset (class in modalities.dataloader.dataset) DummyDatasetConfig (class in modalities.dataloader.dataset) DummyGradientClipper (class in modalities.training.gradient_clipping.fsdp_gradient_clipper) DummyGradientClipperConfig (class in modalities.training.gradient_clipping.fsdp_gradient_clipper_config) DummyLRScheduler (class in modalities.optimizers.lr_schedulers) DummyLRSchedulerConfig (class in modalities.config.config) DummyProgressSubscriber (class in modalities.logging_broker.subscriber_impl.progress_subscriber) DummyProgressSubscriberConfig (class in modalities.config.config) DummyResultSubscriber (class in modalities.logging_broker.subscriber_impl.results_subscriber) DummyResultSubscriberConfig (class in modalities.config.config) DummySampleConfig (class in modalities.dataloader.dataset) DummySampleDataType (class in modalities.dataloader.dataset)

E

elementwise_affine (modalities.models.components.layer_norms.LayerNormConfig attribute) EmbeddedStreamData (class in modalities.dataloader.create_packed_data) EmptySampleError enable_loss_parallel (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) end_factor (modalities.config.config.LinearLRSchedulerConfig attribute) enforce_file_existence_policy() (in module modalities.api) enforce_last_step_checkpointed (modalities.config.instantiation_models.ConsistencyEnforcement attribute) enforce_last_step_evaluated (modalities.config.instantiation_models.ConsistencyEnforcement attribute) enforce_last_step_logged (modalities.config.instantiation_models.ConsistencyEnforcement attribute) enforce_tokens_per_step_consistency (modalities.config.instantiation_models.ConsistencyEnforcement attribute) eod_token (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) (modalities.inference.text.config.TextInferenceComponentConfig attribute) epoch (modalities.config.config.ResumableDistributedSamplerConfig attribute) epochs (modalities.config.config.OneCycleLRSchedulerConfig attribute) eps (modalities.config.config.AdamOptimizerConfig attribute) (modalities.config.config.AdamWOptimizerConfig attribute) (modalities.models.components.layer_norms.LayerNormConfig attribute) epsilon (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.components.layer_norms.RMSLayerNormConfig attribute) epsilon_attn_pool (modalities.models.coca.coca_model.CoCaConfig attribute) ERROR (modalities.api.FileExistencePolicy attribute)

ERROR_MESSAGE (modalities.logging_broker.messages.MessageTypes attribute) eta_min (modalities.config.config.CosineAnnealingLRSchedulerConfig attribute) eval_dataloaders (modalities.config.config.RichProgressSubscriberConfig attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) evaluate() (modalities.evaluator.Evaluator method) evaluate_batch() (modalities.evaluator.Evaluator method) EVALUATION (modalities.logging_broker.messages.ExperimentStatus attribute) evaluation_interval_in_steps (modalities.config.instantiation_models.Intervals attribute) EVALUATION_RESULT (modalities.logging_broker.messages.MessageTypes attribute) evaluation_subscriber (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) EvaluationResultBatch (class in modalities.batch) Evaluator (class in modalities.evaluator) execute_attention() (modalities.models.gpt2.gpt2_model.CausalSelfAttention class method) execute_qkv_transforms() (modalities.models.gpt2.gpt2_model.CausalSelfAttention static method) experiment_id (modalities.config.config.DCPCheckpointSavingConfig attribute) (modalities.config.config.FSDP1CheckpointSavingConfig attribute) (modalities.config.config.WandBEvaluationResultSubscriberConfig attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) experiment_status (modalities.logging_broker.messages.ProgressUpdate attribute) ExperimentStatus (class in modalities.logging_broker.messages) extra (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.Paths.Config attribute)

F

factor (modalities.config.config.ConstantLRSchedulerConfig attribute) ffn_hidden (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) ffn_norm_config (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) FileExistencePolicy (class in modalities.api) final_div_factor (modalities.config.config.OneCycleLRSchedulerConfig attribute) FLOAT (modalities.dataloader.dataset.DummySampleDataType attribute) format_metrics_to_gb() (in module modalities.util) forward() (modalities.conversion.gpt2.modeling_gpt2.GPT2DecoderLayer method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForCausalLM method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForQuestionAnswering method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForSequenceClassification method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForTokenClassification method) (modalities.conversion.gpt2.modeling_gpt2.GPT2Model method) (modalities.conversion.gpt2.modeling_gpt2.LlamaAttention method) (modalities.conversion.gpt2.modeling_gpt2.LlamaFlashAttention2 method) (modalities.conversion.gpt2.modeling_gpt2.LlamaMLP method) (modalities.conversion.gpt2.modeling_gpt2.LlamaRotaryEmbedding method) (modalities.conversion.gpt2.modeling_gpt2.LlamaSdpaAttention method) (modalities.models.coca.attention_pooling.AttentionPooling method) (modalities.models.coca.coca_model.CoCa method) (modalities.models.coca.multi_modal_decoder.MultiModalTextDecoder method) (modalities.models.coca.multi_modal_decoder.TransformerBlock method) (modalities.models.coca.text_decoder.TextDecoder method) (modalities.models.components.layer_norms.RMSLayerNorm method) (modalities.models.gpt2.gpt2_model.CausalSelfAttention method) (modalities.models.gpt2.gpt2_model.GPT2Block method) (modalities.models.gpt2.gpt2_model.GPT2LLM method) (modalities.models.gpt2.gpt2_model.IdentityTransform method) (modalities.models.gpt2.gpt2_model.QueryKeyValueTransform method) (modalities.models.gpt2.gpt2_model.RotaryTransform method) (modalities.models.gpt2.gpt2_model.TransformerMLP method) (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModel method) (modalities.models.huggingface_adapters.hf_adapter.HFModelAdapter method) (modalities.models.model.NNModel method) (modalities.models.model.SwiGLU method) (modalities.models.vision_transformer.vision_transformer_model.ImagePatchEmbedding method) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformer method) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerBlock method) (modalities.nn.attention.MultiHeadAttention method) (modalities.nn.mlp.MLP method)

FORWARD_BACKWARD_TIME (modalities.trainer.ThroughputAggregationKeys attribute) forward_images() (modalities.models.vision_transformer.vision_transformer_model.VisionTransformer method) forward_impl() (modalities.models.gpt2.gpt2_model.GPT2LLM method) FP16 (modalities.config.config.PrecisionEnum attribute) FP32 (modalities.config.config.PrecisionEnum attribute) FP_16 (modalities.running_env.env_utils.MixedPrecisionSettings attribute) (modalities.running_env.env_utils.PyTorchDtypes attribute) FP_32 (modalities.running_env.env_utils.MixedPrecisionSettings attribute) (modalities.running_env.env_utils.PyTorchDtypes attribute) FSDP1CheckpointedModelConfig (class in modalities.config.config) FSDP1CheckpointedOptimizerConfig (class in modalities.config.config) FSDP1CheckpointLoading (class in modalities.checkpointing.fsdp.fsdp_checkpoint_loading) FSDP1CheckpointLoadingConfig (class in modalities.config.config) FSDP1CheckpointLoadingIF (class in modalities.checkpointing.checkpoint_loading) FSDP1CheckpointSaving (class in modalities.checkpointing.fsdp.fsdp_checkpoint_saving) FSDP1CheckpointSavingConfig (class in modalities.config.config) FSDP1GradientClipper (class in modalities.training.gradient_clipping.fsdp_gradient_clipper) FSDP1LoggingOnlyGradientClipper (class in modalities.training.gradient_clipping.fsdp_gradient_clipper) FSDP2GradientClipper (class in modalities.training.gradient_clipping.fsdp_gradient_clipper) FSDP2LoggingOnlyGradientClipper (class in modalities.training.gradient_clipping.fsdp_gradient_clipper) FSDP2MixedPrecisionSettings (class in modalities.running_env.env_utils) FSDP2WrappedModelConfig (class in modalities.config.config) fsdp_block_names (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModel property) FSDPAutoWrapFactoryIF (class in modalities.running_env.fsdp.fsdp_auto_wrapper) FSDPAutoWrapFactoryTypes (class in modalities.running_env.fsdp.fsdp_auto_wrapper) FSDPDummyGradientClipperConfig (class in modalities.training.gradient_clipping.fsdp_gradient_clipper_config) FSDPGradientClipperConfig (class in modalities.training.gradient_clipping.fsdp_gradient_clipper_config) FSDPTransformerAutoWrapPolicyFactory (class in modalities.running_env.fsdp.fsdp_auto_wrapper) (modalities.running_env.fsdp.fsdp_auto_wrapper.FSDPAutoWrapFactoryTypes attribute) FSDPWrappedModelConfig (class in modalities.config.config) fullgraph (modalities.config.config.CompiledModelConfig attribute)

G

gamma (modalities.config.config.StepLRSchedulerConfig attribute) GELU (modalities.models.model.ActivationType attribute) generate_text() (in module modalities.api) (in module modalities.inference.inference) generate_tokens() (modalities.inference.text.inference_component.TextInferenceComponent method) get_activation_checkpointed_fsdp1_model() (modalities.models.model_factory.ModelFactory static method) get_adam() (modalities.optimizers.optimizer_factory.OptimizerFactory method) get_adam_w() (modalities.optimizers.optimizer_factory.OptimizerFactory method) get_all_reduced_value() (modalities.util.Aggregator method) get_auto_wrap_policy() (modalities.running_env.fsdp.fsdp_auto_wrapper.FSDPAutoWrapFactoryIF method) (modalities.running_env.fsdp.fsdp_auto_wrapper.FSDPTransformerAutoWrapPolicyFactory method) get_checkpoint_instruction() (modalities.checkpointing.checkpoint_saving_strategies.CheckpointSavingStrategyIF method) (modalities.checkpointing.checkpoint_saving_strategies.SaveEveryKStepsCheckpointingStrategy method) (modalities.checkpointing.checkpoint_saving_strategies.SaveKMostRecentCheckpointsStrategy method) get_combined_dataset() (modalities.dataloader.dataset_factory.DatasetFactory static method) get_compiled_model() (modalities.models.model_factory.ModelFactory static method) get_component() (modalities.registry.registry.Registry method) get_composed_model_initializer() (modalities.nn.model_initialization.composed_initialization.ComposedInitializationRoutines static method) get_config() (modalities.registry.registry.Registry method) get_dataloader() (modalities.dataloader.dataloader_factory.DataloaderFactory static method) get_dcp_checkpointed_app_state_() (modalities.checkpointing.stateful.app_state_factory.AppStateFactory static method) get_decoder() (modalities.conversion.gpt2.modeling_gpt2.GPT2ForCausalLM method) get_device_mesh() (in module modalities.running_env.fsdp.device_mesh) get_dummy_dataset() (modalities.dataloader.dataset_factory.DatasetFactory static method) get_dummy_progress_subscriber() (modalities.logging_broker.subscriber_impl.subscriber_factory.ProgressSubscriberFactory static method) get_dummy_result_subscriber() (modalities.logging_broker.subscriber_impl.subscriber_factory.ResultsSubscriberFactory static method) get_experiment_id_of_run() (in module modalities.util) get_fsdp1_checkpointed_model() (modalities.models.model_factory.ModelFactory static method) get_fsdp1_checkpointed_optimizer_() (modalities.optimizers.optimizer_factory.OptimizerFactory static method) get_fsdp1_wrapped_model() (modalities.models.model_factory.ModelFactory method) get_fsdp2_wrapped_model() (modalities.models.model_factory.ModelFactory static method) get_global_num_seen_tokens_from_checkpoint_path() (modalities.utils.number_conversion.NumberConversion static method) get_global_num_target_tokens_from_checkpoint_path() (modalities.utils.number_conversion.NumberConversion static method) get_gpt2_model() (modalities.models.model_factory.GPT2ModelFactory static method) get_input_embeddings() (modalities.conversion.gpt2.modeling_gpt2.GPT2ForCausalLM method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForQuestionAnswering method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForSequenceClassification method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForTokenClassification method) (modalities.conversion.gpt2.modeling_gpt2.GPT2Model method) get_jsonl_file_chunk() (modalities.preprocessing.create_chunks.Chunking static method) get_last_step_from_checkpoint_path() (modalities.utils.number_conversion.NumberConversion static method) get_local_num_batches_from_num_samples() (modalities.utils.number_conversion.NumberConversion static method) get_local_num_batches_from_num_tokens() (modalities.utils.number_conversion.NumberConversion static method) get_local_number_of_trainable_parameters() (in module modalities.util) get_logger() (in module modalities.utils.logging) get_lr() (modalities.optimizers.lr_schedulers.DummyLRScheduler method) get_mem_map_dataset() (modalities.dataloader.dataset_factory.DatasetFactory static method) get_model_from_config() (in module modalities.models.utils) get_model_initializer_wrapper() (modalities.nn.model_initialization.composed_initialization.ComposedInitializationRoutines static method) get_module_class_from_name() (in module modalities.util) get_num_samples_from_num_tokens() (modalities.utils.number_conversion.NumberConversion static method) get_num_seen_steps_from_checkpoint_path() (modalities.utils.number_conversion.NumberConversion static method) get_num_steps_from_num_samples() (modalities.utils.number_conversion.NumberConversion static method) get_num_steps_from_num_tokens() (modalities.utils.number_conversion.NumberConversion static method) get_num_steps_from_raw_dataset_index() (modalities.utils.number_conversion.NumberConversion static method) get_num_target_steps_from_checkpoint_path() (modalities.utils.number_conversion.NumberConversion static method) get_num_tokens_from_num_steps() (modalities.utils.number_conversion.NumberConversion static method) get_num_tokens_from_packed_mem_map_dataset_continuous() (modalities.utils.number_conversion.NumberConversion static method) get_optimizer_groups() (in module modalities.optimizers.optimizer_factory) get_output_embeddings() (modalities.conversion.gpt2.modeling_gpt2.GPT2ForCausalLM method) get_packed_mem_map_dataset_continuous() (modalities.dataloader.dataset_factory.DatasetFactory static method) get_packed_mem_map_dataset_megatron() (modalities.dataloader.dataset_factory.DatasetFactory static method) get_parameters() (modalities.models.model.NNModel method)

get_plain_initialization() (modalities.nn.model_initialization.initialization_routines.InitializationRoutines static method) get_predictions() (modalities.batch.InferenceResultBatch method) get_raw_app_state() (modalities.checkpointing.stateful.app_state_factory.AppStateFactory static method) get_raw_index() (modalities.dataloader.dataset_factory.DatasetFactory static method) get_report() (modalities.config.instantiation_models.TrainingReportGenerator method) get_rich_progress_subscriber() (modalities.logging_broker.subscriber_impl.subscriber_factory.ProgressSubscriberFactory static method) get_rich_result_subscriber() (modalities.logging_broker.subscriber_impl.subscriber_factory.ResultsSubscriberFactory static method) get_scaled_embed_initialization() (modalities.nn.model_initialization.initialization_routines.InitializationRoutines static method) get_scaled_initialization() (modalities.nn.model_initialization.initialization_routines.InitializationRoutines static method) get_state_dict() (modalities.checkpointing.stateful.app_state.LRSchedulerStateRetriever static method) (modalities.checkpointing.stateful.app_state.ModelStateRetriever static method) (modalities.checkpointing.stateful.app_state.OptimizerStateRetriever static method) (modalities.checkpointing.stateful.app_state.StateRetrieverIF static method) get_targets() (modalities.batch.InferenceResultBatch method) get_token_id() (modalities.tokenization.tokenizer_wrapper.PreTrainedHFTokenizer method) (modalities.tokenization.tokenizer_wrapper.PreTrainedSPTokenizer method) (modalities.tokenization.tokenizer_wrapper.TokenizerWrapper method) get_tokenized_file_chunk() (modalities.preprocessing.create_chunks.Chunking static method) get_total_number_of_trainable_parameters() (in module modalities.util) get_wandb_result_subscriber() (modalities.logging_broker.subscriber_impl.subscriber_factory.ResultsSubscriberFactory static method) get_weight_initialized_model() (modalities.models.model_factory.ModelFactory static method) global_num_samples (modalities.utils.number_conversion.LocalNumBatchesFromNumSamplesConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumSamplesConfig attribute) global_num_seen_tokens (modalities.config.instantiation_models.TrainingProgress attribute) global_num_tokens (modalities.utils.number_conversion.LocalNumBatchesFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumTokensConfig attribute) global_rank (modalities.config.config.DCPCheckpointLoadingConfig attribute) (modalities.config.config.DCPCheckpointSavingConfig attribute) (modalities.config.config.FSDP1CheckpointLoadingConfig attribute) (modalities.config.config.FSDP1CheckpointSavingConfig attribute) (modalities.config.config.RichProgressSubscriberConfig attribute) (modalities.config.config.RichResultSubscriberConfig attribute) (modalities.config.config.WandBEvaluationResultSubscriberConfig attribute) (modalities.config.instantiation_models.CudaEnvSettings attribute) (modalities.logging_broker.messages.Message attribute) GPT2 (modalities.nn.model_initialization.parameter_name_filters.SupportWeightInitModels attribute) GPT2Block (class in modalities.models.gpt2.gpt2_model) GPT2Config (class in modalities.conversion.gpt2.configuration_gpt2) GPT2DecoderLayer (class in modalities.conversion.gpt2.modeling_gpt2) GPT2ForCausalLM (class in modalities.conversion.gpt2.modeling_gpt2) GPT2ForQuestionAnswering (class in modalities.conversion.gpt2.modeling_gpt2) GPT2ForSequenceClassification (class in modalities.conversion.gpt2.modeling_gpt2) GPT2ForTokenClassification (class in modalities.conversion.gpt2.modeling_gpt2) GPT2LLM (class in modalities.models.gpt2.gpt2_model) GPT2LLMCollateFn (class in modalities.models.gpt2.collator) GPT2LLMCollateFnConfig (class in modalities.config.config) GPT2LLMConfig (class in modalities.models.gpt2.gpt2_model) GPT2MFUCalculator (class in modalities.utils.mfu) GPT2MFUCalculatorConfig (class in modalities.config.config) GPT2Model (class in modalities.conversion.gpt2.modeling_gpt2) GPT2ModelFactory (class in modalities.models.model_factory) GPT2PreTrainedModel (class in modalities.conversion.gpt2.modeling_gpt2) GPT2TokenizerFast (modalities.config.config.TokenizerTypes attribute) gradient_accumulation_steps (modalities.config.instantiation_models.StepProfile attribute) (modalities.utils.number_conversion.NumStepsFromNumSamplesConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromRawDatasetIndexConfig attribute) (modalities.utils.number_conversion.NumTokensFromNumStepsConfig attribute) (modalities.utils.number_conversion.NumTokensFromPackedMemMapDatasetContinuousConfig attribute) gradient_clipper (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) GradientClipperIF (class in modalities.training.gradient_clipping.gradient_clipper) GradientClippingMode (class in modalities.training.gradient_clipping.fsdp_gradient_clipper) Gym (class in modalities.gym)

H

has_bfloat_support() (in module modalities.running_env.env_utils) HEADER_SIZE_IN_BYTES (modalities.dataloader.create_packed_data.EmbeddedStreamData attribute) (modalities.dataloader.dataset.PackedMemMapDatasetBase attribute) HFModelAdapter (class in modalities.models.huggingface_adapters.hf_adapter) HFModelAdapterConfig (class in modalities.models.huggingface_adapters.hf_adapter) hidden_dim (modalities.nn.model_initialization.composed_initialization.ComposedModelInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.PlainInitializationConfig attribute)

hidden_states (modalities.models.huggingface_adapters.hf_adapter.ModalitiesModelOutput attribute) HIGH_LEVEL_PROGRESS_UPDATE (modalities.logging_broker.messages.MessageTypes attribute) hugging_face (modalities.utils.verify_tokenization_consistency.TokenizerTypes attribute) huggingface_prediction_subscription_key (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) HuggingFaceModelTypes (class in modalities.models.huggingface.huggingface_model) HuggingFacePretrainedModel (class in modalities.models.huggingface.huggingface_model) HuggingFacePretrainedModelConfig (class in modalities.models.huggingface.huggingface_model)

I

IdentityTransform (class in modalities.models.gpt2.gpt2_model) (modalities.models.gpt2.gpt2_model.QueryKeyValueTransformType attribute) ImagePatchEmbedding (class in modalities.models.vision_transformer.vision_transformer_model) img_size (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) index_base (modalities.dataloader.create_packed_data.EmbeddedStreamData property) index_path (modalities.config.config.MemMapDatasetConfig attribute) (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) IndexGenerator (class in modalities.dataloader.create_index) InferenceResultBatch (class in modalities.batch) InitializationRoutines (class in modalities.nn.model_initialization.initialization_routines) initialize_in_place() (modalities.nn.model_initialization.composed_initialization.ModelInitializerWrapper method) (modalities.nn.model_initialization.initialization_if.ModelInitializationIF method) (modalities.nn.model_initialization.initialization_routines.NamedParameterwiseNormalInitialization method)

instance_key (modalities.config.config.ReferenceConfig attribute) INT (modalities.dataloader.dataset.DummySampleDataType attribute) Intervals (class in modalities.config.instantiation_models) intervals (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) is_loaded (modalities.checkpointing.stateful.app_state.AppState property) is_module_to_apply_activation_checkpointing() (in module modalities.training.activation_checkpointing) is_running_with_torchrun() (in module modalities.running_env.env_utils) is_special_token_id() (modalities.tokenization.tokenizer_wrapper.PreTrainedHFTokenizer method) (modalities.tokenization.tokenizer_wrapper.PreTrainedSPTokenizer method) (modalities.tokenization.tokenizer_wrapper.TokenizerWrapper method)

J

join_embedded_stream_data() (in module modalities.dataloader.create_packed_data)

jq_pattern (modalities.config.config.MemMapDatasetConfig attribute) (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute)

K

k (modalities.config.config.SaveEveryKStepsCheckpointingStrategyConfig attribute) (modalities.config.config.SaveKMostRecentCheckpointsStrategyConfig attribute)

keys_to_ignore_at_inference (modalities.conversion.gpt2.configuration_gpt2.GPT2Config attribute) kwargs (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) KwargsForCausalLM (class in modalities.conversion.gpt2.modeling_gpt2)

L

LargeFileLinesReader (class in modalities.dataloader.large_file_lines_reader) last_epoch (modalities.config.config.ConstantLRSchedulerConfig attribute) (modalities.config.config.CosineAnnealingLRSchedulerConfig attribute) (modalities.config.config.LinearLRSchedulerConfig attribute) (modalities.config.config.OneCycleLRSchedulerConfig attribute) (modalities.config.config.StepLRSchedulerConfig attribute) last_step (modalities.config.instantiation_models.TrainingProgress attribute) layer_norm (modalities.models.gpt2.gpt2_model.LayerNorms attribute) LayerNormConfig (class in modalities.models.components.layer_norms) LayerNorms (class in modalities.models.gpt2.gpt2_model) LayerNormWrapperConfig (class in modalities.models.gpt2.gpt2_model) LinearLRSchedulerConfig (class in modalities.config.config) LlamaAttention (class in modalities.conversion.gpt2.modeling_gpt2) LlamaDynamicNTKScalingRotaryEmbedding (class in modalities.conversion.gpt2.modeling_gpt2) LlamaFlashAttention2 (class in modalities.conversion.gpt2.modeling_gpt2) LlamaLinearScalingRotaryEmbedding (class in modalities.conversion.gpt2.modeling_gpt2) LlamaMLP (class in modalities.conversion.gpt2.modeling_gpt2) LlamaRotaryEmbedding (class in modalities.conversion.gpt2.modeling_gpt2) LlamaSdpaAttention (class in modalities.conversion.gpt2.modeling_gpt2) LlamaTokenizerFast (modalities.config.config.TokenizerTypes attribute) LLMDataLoader (class in modalities.dataloader.dataloader) LLMDataLoaderConfig (class in modalities.config.config) lm_head_norm_config (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) load_app_config_dict() (in module modalities.config.config) load_checkpoint_() (modalities.checkpointing.checkpoint_loading.DistributedCheckpointLoadingIF method) (modalities.checkpointing.fsdp.fsdp_checkpoint_loading.DCPCheckpointLoading method) load_model_checkpoint() (modalities.checkpointing.checkpoint_loading.FSDP1CheckpointLoadingIF method) (modalities.checkpointing.fsdp.fsdp_checkpoint_loading.FSDP1CheckpointLoading method) (modalities.checkpointing.torch.torch_checkpoint_loading.TorchCheckpointLoading method) load_optimizer_checkpoint_() (modalities.checkpointing.checkpoint_loading.FSDP1CheckpointLoadingIF method) (modalities.checkpointing.fsdp.fsdp_checkpoint_loading.FSDP1CheckpointLoading method) (modalities.checkpointing.torch.torch_checkpoint_loading.TorchCheckpointLoading method)

load_state_dict() (modalities.checkpointing.stateful.app_state.AppState method) load_state_dict_() (modalities.checkpointing.stateful.app_state.LRSchedulerStateRetriever static method) (modalities.checkpointing.stateful.app_state.ModelStateRetriever static method) (modalities.checkpointing.stateful.app_state.OptimizerStateRetriever static method) (modalities.checkpointing.stateful.app_state.StateRetrieverIF static method) local_micro_batch_size (modalities.utils.number_conversion.LocalNumBatchesFromNumSamplesConfig attribute) (modalities.utils.number_conversion.LocalNumBatchesFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumSamplesConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromRawDatasetIndexConfig attribute) (modalities.utils.number_conversion.NumTokensFromNumStepsConfig attribute) (modalities.utils.number_conversion.NumTokensFromPackedMemMapDatasetContinuousConfig attribute) local_rank (modalities.config.instantiation_models.CudaEnvSettings attribute) (modalities.logging_broker.messages.Message attribute) local_train_micro_batch_size (modalities.config.instantiation_models.StepProfile attribute) LocalNumBatchesFromNumSamplesConfig (class in modalities.utils.number_conversion) LocalNumBatchesFromNumTokensConfig (class in modalities.utils.number_conversion) logits (modalities.models.huggingface_adapters.hf_adapter.ModalitiesModelOutput attribute) LookupEnum (class in modalities.config.lookup_enum) Loss (class in modalities.loss_functions) loss_fn (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) losses (modalities.batch.EvaluationResultBatch attribute) lr (modalities.config.config.AdamOptimizerConfig attribute) (modalities.config.config.AdamWOptimizerConfig attribute) lr_scheduler (modalities.checkpointing.stateful.app_state.AppState property) LR_SCHEDULER (modalities.checkpointing.stateful.app_state.StatefulComponents attribute) lr_scheduler (modalities.config.config.RawAppStateConfig attribute) LRSchedulerStateRetriever (class in modalities.checkpointing.stateful.app_state)

M

MANUAL (modalities.models.gpt2.gpt2_model.AttentionImplementation attribute) manual_scaled_dot_product_attention() (in module modalities.models.gpt2.gpt2_model) max_length (modalities.config.config.PreTrainedHFTokenizerConfig attribute) max_length_k (modalities.conversion.gpt2.modeling_gpt2.KwargsForCausalLM attribute) max_length_q (modalities.conversion.gpt2.modeling_gpt2.KwargsForCausalLM attribute) max_lr (modalities.config.config.OneCycleLRSchedulerConfig attribute) max_momentum (modalities.config.config.OneCycleLRSchedulerConfig attribute) MAX_NORM (modalities.training.gradient_clipping.fsdp_gradient_clipper.GradientClippingMode attribute) max_norm (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.FSDPGradientClipperConfig attribute) mean (modalities.nn.model_initialization.composed_initialization.ComposedModelInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.PlainInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledEmbedInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledInitializationConfig attribute) MemMapDataset (class in modalities.dataloader.dataset) MemMapDatasetConfig (class in modalities.config.config) merge_packed_data_files() (in module modalities.api) Message (class in modalities.logging_broker.messages) message_type (modalities.logging_broker.messages.Message attribute) MessageBroker (class in modalities.logging_broker.message_broker) MessageBrokerIF (class in modalities.logging_broker.message_broker) MessagePublisher (class in modalities.logging_broker.publisher) MessagePublisherIF (class in modalities.logging_broker.publisher) MessageSubscriberIF (class in modalities.logging_broker.subscriber) MessageTypes (class in modalities.logging_broker.messages) metrics (modalities.batch.EvaluationResultBatch attribute) mfu_calculator (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) MFUCalculatorABC (class in modalities.utils.mfu) MIXED_PRECISION_MEGATRON (modalities.running_env.env_utils.MixedPrecisionSettings attribute) mixed_precision_settings (modalities.config.config.FSDP1CheckpointLoadingConfig attribute) (modalities.config.config.FSDP2WrappedModelConfig attribute) (modalities.config.config.FSDPWrappedModelConfig attribute) MixedPrecisionSettings (class in modalities.running_env.env_utils) MLP (class in modalities.nn.mlp) modalities module modalities.api module modalities.batch module modalities.checkpointing module modalities.checkpointing.checkpoint_conversion module modalities.checkpointing.checkpoint_loading module modalities.checkpointing.checkpoint_saving module modalities.checkpointing.checkpoint_saving_execution module modalities.checkpointing.checkpoint_saving_instruction module modalities.checkpointing.checkpoint_saving_strategies module modalities.checkpointing.fsdp module modalities.checkpointing.fsdp.fsdp_checkpoint_loading module modalities.checkpointing.fsdp.fsdp_checkpoint_saving module modalities.checkpointing.stateful module modalities.checkpointing.stateful.app_state module modalities.checkpointing.stateful.app_state_factory module modalities.checkpointing.torch module modalities.checkpointing.torch.torch_checkpoint_loading module modalities.config module modalities.config.component_factory module modalities.config.config module modalities.config.instantiation_models module modalities.config.lookup_enum module modalities.config.pydantic_if_types module modalities.config.utils module modalities.conversion module modalities.conversion.gpt2 module modalities.conversion.gpt2.configuration_gpt2 module modalities.conversion.gpt2.conversion_code module modalities.conversion.gpt2.conversion_model module modalities.conversion.gpt2.conversion_tokenizer module modalities.conversion.gpt2.convert_gpt2 module modalities.conversion.gpt2.modeling_gpt2 module modalities.dataloader module modalities.dataloader.create_index module modalities.dataloader.create_packed_data module modalities.dataloader.dataloader module modalities.dataloader.dataloader_factory module modalities.dataloader.dataset module modalities.dataloader.dataset_factory module modalities.dataloader.large_file_lines_reader module modalities.dataloader.preprocessing module modalities.dataloader.samplers module modalities.evaluator module modalities.exceptions module modalities.gym module modalities.inference module modalities.inference.inference module modalities.inference.text module modalities.inference.text.config module modalities.inference.text.inference_component module modalities.logging_broker module modalities.logging_broker.message_broker module modalities.logging_broker.messages module modalities.logging_broker.publisher module modalities.logging_broker.subscriber module modalities.logging_broker.subscriber_impl module modalities.logging_broker.subscriber_impl.progress_subscriber module modalities.logging_broker.subscriber_impl.results_subscriber module modalities.logging_broker.subscriber_impl.subscriber_factory module modalities.loss_functions module modalities.models module modalities.models.coca module modalities.models.coca.attention_pooling module modalities.models.coca.coca_model module modalities.models.coca.collator module modalities.models.coca.multi_modal_decoder module modalities.models.coca.text_decoder module modalities.models.components module modalities.models.components.layer_norms module modalities.models.gpt2 module modalities.models.gpt2.collator module modalities.models.gpt2.gpt2_model module modalities.models.huggingface module modalities.models.huggingface.huggingface_model module modalities.models.huggingface_adapters module modalities.models.huggingface_adapters.hf_adapter module modalities.models.model module modalities.models.model_factory module modalities.models.utils module modalities.models.vision_transformer module modalities.models.vision_transformer.vision_transformer_model module modalities.nn module modalities.nn.attention module modalities.nn.mlp module modalities.nn.model_initialization module modalities.nn.model_initialization.composed_initialization module modalities.nn.model_initialization.initialization_if module modalities.nn.model_initialization.initialization_routines module modalities.nn.model_initialization.parameter_name_filters module modalities.optimizers module modalities.optimizers.lr_schedulers module modalities.optimizers.optimizer_factory module modalities.preprocessing module modalities.preprocessing.create_chunks module modalities.preprocessing.shuffle_data module modalities.registry module modalities.registry.components module modalities.registry.registry module modalities.running_env module modalities.running_env.cuda_env module modalities.running_env.env_utils module modalities.running_env.fsdp module modalities.running_env.fsdp.device_mesh module modalities.running_env.fsdp.fsdp_auto_wrapper module modalities.running_env.fsdp.reducer module modalities.tokenization module modalities.tokenization.tokenizer_wrapper module modalities.trainer module modalities.training module modalities.training.activation_checkpointing module modalities.training.gradient_clipping module modalities.training.gradient_clipping.fsdp_gradient_clipper module modalities.training.gradient_clipping.fsdp_gradient_clipper_config module modalities.training.gradient_clipping.gradient_clipper module modalities.training.training_progress module modalities.util module modalities.utils module

modalities.utils.logging module modalities.utils.mfu module modalities.utils.number_conversion module modalities.utils.seeding module modalities.utils.typing module modalities.utils.verify_tokenization_consistency module ModalitiesModelOutput (class in modalities.models.huggingface_adapters.hf_adapter) mode (modalities.config.config.WandBEvaluationResultSubscriberConfig attribute) MODEL (modalities.checkpointing.fsdp.fsdp_checkpoint_saving.CheckpointingEntityType attribute) model (modalities.checkpointing.stateful.app_state.AppState property) MODEL (modalities.checkpointing.stateful.app_state.StatefulComponents attribute) model (modalities.config.config.ActivationCheckpointedModelConfig attribute) (modalities.config.config.CompiledModelConfig attribute) (modalities.config.config.FSDP1CheckpointedModelConfig attribute) (modalities.config.config.FSDP2WrappedModelConfig attribute) (modalities.config.config.FSDPWrappedModelConfig attribute) (modalities.config.config.RawAppStateConfig attribute) (modalities.config.config.WeightInitializedModelConfig attribute) (modalities.inference.text.config.TextInferenceComponentConfig attribute) MODEL (modalities.models.utils.ModelTypeEnum attribute) model_args (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) model_checkpoint_path (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.WarmstartCheckpointPaths attribute) model_config (modalities.config.config.ActivationCheckpointedModelConfig attribute) (modalities.config.config.AdamOptimizerConfig attribute) (modalities.config.config.AdamWOptimizerConfig attribute) (modalities.config.config.BatchSamplerConfig attribute) (modalities.config.config.CheckpointSavingConfig attribute) (modalities.config.config.CLMCrossEntropyLossConfig attribute) (modalities.config.config.CombinedDatasetConfig attribute) (modalities.config.config.CompiledModelConfig attribute) (modalities.config.config.ConstantLRSchedulerConfig attribute) (modalities.config.config.CosineAnnealingLRSchedulerConfig attribute) (modalities.config.config.DCPAppStateConfig attribute) (modalities.config.config.DCPCheckpointLoadingConfig attribute) (modalities.config.config.DCPCheckpointSavingConfig attribute) (modalities.config.config.DistributedSamplerConfig attribute) (modalities.config.config.DummyLRSchedulerConfig attribute) (modalities.config.config.DummyProgressSubscriberConfig attribute) (modalities.config.config.DummyResultSubscriberConfig attribute) (modalities.config.config.FSDP1CheckpointedModelConfig attribute) (modalities.config.config.FSDP1CheckpointedOptimizerConfig attribute) (modalities.config.config.FSDP1CheckpointLoadingConfig attribute) (modalities.config.config.FSDP1CheckpointSavingConfig attribute) (modalities.config.config.FSDP2WrappedModelConfig attribute) (modalities.config.config.FSDPWrappedModelConfig attribute) (modalities.config.config.GPT2LLMCollateFnConfig attribute) (modalities.config.config.GPT2MFUCalculatorConfig attribute) (modalities.config.config.LinearLRSchedulerConfig attribute) (modalities.config.config.LLMDataLoaderConfig attribute) (modalities.config.config.MemMapDatasetConfig attribute) (modalities.config.config.OneCycleLRSchedulerConfig attribute) (modalities.config.config.PackedMemMapDatasetContinuousConfig attribute) (modalities.config.config.PackedMemMapDatasetMegatronConfig attribute) (modalities.config.config.PreTrainedHFTokenizerConfig attribute) (modalities.config.config.PreTrainedSPTokenizerConfig attribute) (modalities.config.config.RawAppStateConfig attribute) (modalities.config.config.ReferenceConfig attribute) (modalities.config.config.ResumableDistributedSamplerConfig attribute) (modalities.config.config.RichProgressSubscriberConfig attribute) (modalities.config.config.RichResultSubscriberConfig attribute) (modalities.config.config.SaveEveryKStepsCheckpointingStrategyConfig attribute) (modalities.config.config.SaveKMostRecentCheckpointsStrategyConfig attribute) (modalities.config.config.SequentialSamplerConfig attribute) (modalities.config.config.StepLRSchedulerConfig attribute) (modalities.config.config.TorchCheckpointLoadingConfig attribute) (modalities.config.config.WandBEvaluationResultSubscriberConfig attribute) (modalities.config.config.WeightInitializedModelConfig attribute) (modalities.config.instantiation_models.ConsistencyEnforcement attribute) (modalities.config.instantiation_models.CudaEnvSettings attribute) (modalities.config.instantiation_models.Intervals attribute) (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel attribute) (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) (modalities.config.instantiation_models.StepProfile attribute) (modalities.config.instantiation_models.TextGenerationInstantiationModel attribute) (modalities.config.instantiation_models.TextGenerationInstantiationModel.TextGenerationSettings attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.DCPWarmstartCheckpointPaths attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.Paths attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.WarmstartCheckpointPaths attribute) (modalities.config.instantiation_models.TrainingProgress attribute) (modalities.config.instantiation_models.TrainingTarget attribute) (modalities.dataloader.dataset.DummyDatasetConfig attribute) (modalities.dataloader.dataset.DummySampleConfig attribute) (modalities.inference.text.config.TextInferenceComponentConfig attribute) (modalities.models.coca.coca_model.CoCaConfig attribute) (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.coca.collator.CoCaCollateFnConfig attribute) (modalities.models.components.layer_norms.LayerNormConfig attribute) (modalities.models.components.layer_norms.RMSLayerNormConfig attribute) (modalities.models.gpt2.gpt2_model.AttentionConfig attribute) (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig attribute) (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig.IdentityTransformConfig attribute) (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig.RotaryTransformConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.gpt2.gpt2_model.LayerNormWrapperConfig attribute) (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) (modalities.nn.attention.AttentionConfig attribute) (modalities.nn.model_initialization.composed_initialization.ComposedModelInitializationConfig attribute) (modalities.nn.model_initialization.composed_initialization.ModelInitializerWrapperConfig attribute) (modalities.nn.model_initialization.initialization_routines.PlainInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledEmbedInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledInitializationConfig attribute) (modalities.nn.model_initialization.parameter_name_filters.RegexFilter attribute) (modalities.running_env.env_utils.FSDP2MixedPrecisionSettings attribute) (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.DummyGradientClipperConfig attribute) (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.FSDPDummyGradientClipperConfig attribute) (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.FSDPGradientClipperConfig attribute) (modalities.utils.number_conversion.LocalNumBatchesFromNumSamplesConfig attribute) (modalities.utils.number_conversion.LocalNumBatchesFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumberConversionFromCheckpointPathConfig attribute) (modalities.utils.number_conversion.NumSamplesFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumSamplesConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromRawDatasetIndexConfig attribute) (modalities.utils.number_conversion.NumTokensFromNumStepsConfig attribute) (modalities.utils.number_conversion.NumTokensFromPackedMemMapDatasetContinuousConfig attribute) model_initializer (modalities.config.config.WeightInitializedModelConfig attribute) model_initializers (modalities.nn.model_initialization.composed_initialization.ModelInitializerWrapperConfig attribute) model_name (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) model_path (modalities.config.instantiation_models.TextGenerationInstantiationModel.TextGenerationSettings attribute) model_predict_batch() (in module modalities.models.model) model_raw (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) model_type (modalities.conversion.gpt2.configuration_gpt2.GPT2Config attribute) (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) (modalities.models.huggingface_adapters.hf_adapter.HFModelAdapterConfig attribute) (modalities.nn.model_initialization.composed_initialization.ComposedModelInitializationConfig attribute) ModelFactory (class in modalities.models.model_factory) ModelInitializationIF (class in modalities.nn.model_initialization.initialization_if) ModelInitializerWrapper (class in modalities.nn.model_initialization.composed_initialization) ModelInitializerWrapperConfig (class in modalities.nn.model_initialization.composed_initialization) ModelStateError ModelStateRetriever (class in modalities.checkpointing.stateful.app_state) ModelTypeEnum (class in modalities.models.utils) module modalities modalities.api modalities.batch modalities.checkpointing modalities.checkpointing.checkpoint_conversion modalities.checkpointing.checkpoint_loading modalities.checkpointing.checkpoint_saving modalities.checkpointing.checkpoint_saving_execution modalities.checkpointing.checkpoint_saving_instruction modalities.checkpointing.checkpoint_saving_strategies modalities.checkpointing.fsdp modalities.checkpointing.fsdp.fsdp_checkpoint_loading modalities.checkpointing.fsdp.fsdp_checkpoint_saving modalities.checkpointing.stateful modalities.checkpointing.stateful.app_state modalities.checkpointing.stateful.app_state_factory modalities.checkpointing.torch modalities.checkpointing.torch.torch_checkpoint_loading modalities.config modalities.config.component_factory modalities.config.config modalities.config.instantiation_models modalities.config.lookup_enum modalities.config.pydantic_if_types modalities.config.utils modalities.conversion modalities.conversion.gpt2 modalities.conversion.gpt2.configuration_gpt2 modalities.conversion.gpt2.conversion_code modalities.conversion.gpt2.conversion_model modalities.conversion.gpt2.conversion_tokenizer modalities.conversion.gpt2.convert_gpt2 modalities.conversion.gpt2.modeling_gpt2 modalities.dataloader modalities.dataloader.create_index modalities.dataloader.create_packed_data modalities.dataloader.dataloader modalities.dataloader.dataloader_factory modalities.dataloader.dataset modalities.dataloader.dataset_factory modalities.dataloader.large_file_lines_reader modalities.dataloader.preprocessing modalities.dataloader.samplers modalities.evaluator modalities.exceptions modalities.gym modalities.inference modalities.inference.inference modalities.inference.text modalities.inference.text.config modalities.inference.text.inference_component modalities.logging_broker modalities.logging_broker.message_broker modalities.logging_broker.messages modalities.logging_broker.publisher modalities.logging_broker.subscriber modalities.logging_broker.subscriber_impl modalities.logging_broker.subscriber_impl.progress_subscriber modalities.logging_broker.subscriber_impl.results_subscriber modalities.logging_broker.subscriber_impl.subscriber_factory modalities.loss_functions modalities.models modalities.models.coca modalities.models.coca.attention_pooling modalities.models.coca.coca_model modalities.models.coca.collator modalities.models.coca.multi_modal_decoder modalities.models.coca.text_decoder modalities.models.components modalities.models.components.layer_norms modalities.models.gpt2 modalities.models.gpt2.collator modalities.models.gpt2.gpt2_model modalities.models.huggingface modalities.models.huggingface.huggingface_model modalities.models.huggingface_adapters modalities.models.huggingface_adapters.hf_adapter modalities.models.model modalities.models.model_factory modalities.models.utils modalities.models.vision_transformer modalities.models.vision_transformer.vision_transformer_model modalities.nn modalities.nn.attention modalities.nn.mlp modalities.nn.model_initialization modalities.nn.model_initialization.composed_initialization modalities.nn.model_initialization.initialization_if modalities.nn.model_initialization.initialization_routines modalities.nn.model_initialization.parameter_name_filters modalities.optimizers modalities.optimizers.lr_schedulers modalities.optimizers.optimizer_factory modalities.preprocessing modalities.preprocessing.create_chunks modalities.preprocessing.shuffle_data modalities.registry modalities.registry.components modalities.registry.registry modalities.running_env modalities.running_env.cuda_env modalities.running_env.env_utils modalities.running_env.fsdp modalities.running_env.fsdp.device_mesh modalities.running_env.fsdp.fsdp_auto_wrapper modalities.running_env.fsdp.reducer modalities.tokenization modalities.tokenization.tokenizer_wrapper modalities.trainer modalities.training modalities.training.activation_checkpointing modalities.training.gradient_clipping modalities.training.gradient_clipping.fsdp_gradient_clipper modalities.training.gradient_clipping.fsdp_gradient_clipper_config modalities.training.gradient_clipping.gradient_clipper modalities.training.training_progress modalities.util modalities.utils modalities.utils.logging modalities.utils.mfu modalities.utils.number_conversion modalities.utils.seeding modalities.utils.typing modalities.utils.verify_tokenization_consistency MultiHeadAttention (class in modalities.nn.attention) MultiModalTextDecoder (class in modalities.models.coca.multi_modal_decoder)

N

n_classes (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) n_embd (modalities.config.config.GPT2MFUCalculatorConfig attribute) (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig.RotaryTransformConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) n_head (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig.RotaryTransformConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) n_head_kv (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) n_head_q (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) n_img_channels (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) n_layer (modalities.config.config.GPT2MFUCalculatorConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) n_layer_multimodal_text (modalities.models.coca.coca_model.TextDecoderConfig attribute) n_layer_text (modalities.models.coca.coca_model.TextDecoderConfig attribute) n_pool_head (modalities.models.coca.coca_model.CoCaConfig attribute) n_vision_queries (modalities.models.coca.coca_model.CoCaConfig attribute) NamedParameterwiseNormalInitialization (class in modalities.nn.model_initialization.initialization_routines) nccl (modalities.config.config.ProcessGroupBackendType attribute) nce_loss() (in module modalities.loss_functions) NCELoss (class in modalities.loss_functions) ndim (modalities.models.components.layer_norms.RMSLayerNormConfig attribute) NNModel (class in modalities.models.model) NO_MIXED_PRECISION (modalities.running_env.env_utils.MixedPrecisionSettings attribute) NON_CAUSAL_SELF_ATTENTION (modalities.nn.attention.AttentionType attribute) NOPE (modalities.models.gpt2.gpt2_model.PositionTypes attribute) norm_type (modalities.models.gpt2.gpt2_model.LayerNormWrapperConfig attribute) (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.FSDPDummyGradientClipperConfig attribute) (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.FSDPGradientClipperConfig attribute) normalized_shape (modalities.models.components.layer_norms.LayerNormConfig attribute) np_dtype_of_tokens_on_disk_from_bytes (modalities.dataloader.dataset.PackedMemMapDatasetBase attribute) num_cpus (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) num_items_in_batch (modalities.conversion.gpt2.modeling_gpt2.KwargsForCausalLM attribute) num_layers (modalities.nn.model_initialization.composed_initialization.ComposedModelInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledInitializationConfig attribute) num_ranks (modalities.config.config.RichResultSubscriberConfig attribute) (modalities.utils.number_conversion.LocalNumBatchesFromNumSamplesConfig attribute) (modalities.utils.number_conversion.LocalNumBatchesFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumSamplesConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromRawDatasetIndexConfig attribute) (modalities.utils.number_conversion.NumTokensFromNumStepsConfig attribute) (modalities.utils.number_conversion.NumTokensFromPackedMemMapDatasetContinuousConfig attribute)

num_replicas (modalities.config.config.DistributedSamplerConfig attribute) (modalities.config.config.ResumableDistributedSamplerConfig attribute) num_samples (modalities.dataloader.dataset.DummyDatasetConfig attribute) NUM_SAMPLES (modalities.trainer.ThroughputAggregationKeys attribute) num_seen_samples (modalities.config.instantiation_models.TrainingProgress attribute) num_seen_steps (modalities.config.config.RichProgressSubscriberConfig attribute) (modalities.config.instantiation_models.TrainingProgress attribute) num_seen_steps_current_run (modalities.training.training_progress.TrainingProgress attribute) num_seen_steps_previous_run (modalities.training.training_progress.TrainingProgress attribute) num_seen_steps_total (modalities.training.training_progress.TrainingProgress property) num_seen_tokens_current_run (modalities.training.training_progress.TrainingProgress attribute) num_seen_tokens_previous_run (modalities.training.training_progress.TrainingProgress attribute) num_seen_tokens_total (modalities.training.training_progress.TrainingProgress property) num_steps (modalities.utils.number_conversion.NumTokensFromNumStepsConfig attribute) num_steps_done (modalities.logging_broker.messages.ProgressUpdate attribute) num_target_steps (modalities.config.config.RichProgressSubscriberConfig attribute) (modalities.config.instantiation_models.TrainingTarget attribute) (modalities.training.training_progress.TrainingProgress attribute) num_target_tokens (modalities.config.instantiation_models.TrainingTarget attribute) (modalities.training.training_progress.TrainingProgress attribute) num_tokens (modalities.utils.number_conversion.NumSamplesFromNumTokensConfig attribute) num_train_steps_done (modalities.batch.EvaluationResultBatch attribute) num_workers (modalities.config.config.LLMDataLoaderConfig attribute) NumberConversion (class in modalities.utils.number_conversion) NumberConversionFromCheckpointPathConfig (class in modalities.utils.number_conversion) NumSamplesFromNumTokensConfig (class in modalities.utils.number_conversion) NumStepsFromNumSamplesConfig (class in modalities.utils.number_conversion) NumStepsFromNumTokensConfig (class in modalities.utils.number_conversion) NumStepsFromRawDatasetIndexConfig (class in modalities.utils.number_conversion) NumTokensFromNumStepsConfig (class in modalities.utils.number_conversion) NumTokensFromPackedMemMapDatasetContinuousConfig (class in modalities.utils.number_conversion)

O

OFFLINE (modalities.config.config.WandbMode attribute) OneCycleLRSchedulerConfig (class in modalities.config.config) ONLINE (modalities.config.config.WandbMode attribute) OPTIMIZER (modalities.checkpointing.fsdp.fsdp_checkpoint_saving.CheckpointingEntityType attribute) optimizer (modalities.checkpointing.stateful.app_state.AppState property) OPTIMIZER (modalities.checkpointing.stateful.app_state.StatefulComponents attribute) optimizer (modalities.config.config.ConstantLRSchedulerConfig attribute) (modalities.config.config.CosineAnnealingLRSchedulerConfig attribute) (modalities.config.config.DummyLRSchedulerConfig attribute) (modalities.config.config.FSDP1CheckpointedOptimizerConfig attribute) (modalities.config.config.LinearLRSchedulerConfig attribute) (modalities.config.config.OneCycleLRSchedulerConfig attribute) (modalities.config.config.RawAppStateConfig attribute) (modalities.config.config.StepLRSchedulerConfig attribute)

optimizer_checkpoint_path (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings.WarmstartCheckpointPaths attribute) OptimizerError OptimizerFactory (class in modalities.optimizers.optimizer_factory) OptimizerStateRetriever (class in modalities.checkpointing.stateful.app_state) OVERRIDE (modalities.api.FileExistencePolicy attribute)

P

P1_NORM (modalities.training.gradient_clipping.fsdp_gradient_clipper.GradientClippingMode attribute) P2_NORM (modalities.training.gradient_clipping.fsdp_gradient_clipper.GradientClippingMode attribute) pack_encoded_data() (in module modalities.api) PackedDataGenerator (class in modalities.dataloader.create_packed_data) PackedDatasetComponentsInstantiationModel (class in modalities.config.instantiation_models) PackedDatasetComponentsInstantiationModel.PackedDatasetSettings (class in modalities.config.instantiation_models) PackedMemMapDatasetBase (class in modalities.dataloader.dataset) PackedMemMapDatasetContinuous (class in modalities.dataloader.dataset) PackedMemMapDatasetContinuousConfig (class in modalities.config.config) PackedMemMapDatasetMegatron (class in modalities.dataloader.dataset) PackedMemMapDatasetMegatronConfig (class in modalities.config.config) padding (modalities.config.config.PreTrainedHFTokenizerConfig attribute) ParallelismDegrees (class in modalities.running_env.fsdp.device_mesh) param_dtype (modalities.running_env.env_utils.FSDP2MixedPrecisionSettings attribute) parameter_name_regexes (modalities.nn.model_initialization.initialization_routines.PlainInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledEmbedInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledInitializationConfig attribute) parse_device() (modalities.config.config.TorchCheckpointLoadingConfig class method) (modalities.config.instantiation_models.TextGenerationInstantiationModel.TextGenerationSettings class method) (modalities.inference.text.config.TextInferenceComponentConfig class method) parse_enum_by_name() (in module modalities.util) parse_mixed_precision_setting_by_name() (modalities.config.config.FSDP1CheckpointLoadingConfig class method) (modalities.config.config.FSDPWrappedModelConfig class method) parse_sharding_strategy_by_name() (modalities.config.config.FSDP1CheckpointLoadingConfig class method) (modalities.config.config.FSDPWrappedModelConfig class method) (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig class method) parse_torch_device() (in module modalities.config.utils) pass_type (modalities.config.config.ReferenceConfig attribute) PassType (class in modalities.config.config) patch_size (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) patch_stride (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) paths (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) payload (modalities.logging_broker.messages.Message attribute) pct_start (modalities.config.config.OneCycleLRSchedulerConfig attribute) pin_memory (modalities.config.config.LLMDataLoaderConfig attribute) pipeline_parallel_degree (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute)

PLAIN (modalities.nn.model_initialization.parameter_name_filters.WeightInitTypes attribute) PlainInitializationConfig (class in modalities.nn.model_initialization.initialization_routines) poe_type (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) PositionTypes (class in modalities.models.gpt2.gpt2_model) PP (modalities.running_env.fsdp.device_mesh.ParallelismDegrees attribute) precision (modalities.config.config.TorchCheckpointLoadingConfig attribute) PrecisionEnum (class in modalities.config.config) prediction_key (modalities.config.config.CLMCrossEntropyLossConfig attribute) (modalities.models.coca.coca_model.CoCaConfig attribute) (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) predictions (modalities.batch.InferenceResultBatch attribute) prepare_inputs_for_generation() (modalities.models.huggingface_adapters.hf_adapter.HFModelAdapter method) pretrained_model_name_or_path (modalities.config.config.PreTrainedHFTokenizerConfig attribute) PreTrainedHFTokenizer (class in modalities.tokenization.tokenizer_wrapper) PreTrainedHFTokenizerConfig (class in modalities.config.config) PreTrainedSPTokenizer (class in modalities.tokenization.tokenizer_wrapper) PreTrainedSPTokenizerConfig (class in modalities.config.config) print_rank_0() (in module modalities.util) processed_samples_queue_size (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) ProcessGroupBackendType (class in modalities.config.config) processing_batch_size (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) progress_subscriber (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) ProgressSubscriberFactory (class in modalities.logging_broker.subscriber_impl.subscriber_factory) ProgressUpdate (class in modalities.logging_broker.messages) project (modalities.config.config.WandBEvaluationResultSubscriberConfig attribute) projection() (modalities.models.gpt2.gpt2_model.CausalSelfAttention method) prompt_template (modalities.inference.text.config.TextInferenceComponentConfig attribute) publish_message() (modalities.logging_broker.publisher.MessagePublisher method) (modalities.logging_broker.publisher.MessagePublisherIF method) PydanticThirdPartyTypeIF (class in modalities.config.pydantic_if_types) PYTORCH_FLASH (modalities.models.gpt2.gpt2_model.AttentionImplementation attribute) PYTORCH_FLASH_ATTENTION (modalities.nn.attention.AttentionEngineType attribute) PyTorchDtypes (class in modalities.running_env.env_utils)

Q

qkv_transforms (modalities.models.gpt2.gpt2_model.AttentionConfig attribute)

QueryKeyValueTransform (class in modalities.models.gpt2.gpt2_model) QueryKeyValueTransformType (class in modalities.models.gpt2.gpt2_model)

R

rank (modalities.config.config.DistributedSamplerConfig attribute) (modalities.config.config.ResumableDistributedSamplerConfig attribute) raw_app_state (modalities.config.config.DCPAppStateConfig attribute) raw_data_path (modalities.config.config.MemMapDatasetConfig attribute) (modalities.config.config.PackedMemMapDatasetContinuousConfig attribute) (modalities.config.config.PackedMemMapDatasetMegatronConfig attribute) raw_index_path (modalities.utils.number_conversion.NumStepsFromRawDatasetIndexConfig attribute) raw_samples_queue_size (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) RawAppStateConfig (class in modalities.config.config) reduce() (modalities.running_env.fsdp.reducer.Reducer static method) reduce_dtype (modalities.running_env.env_utils.FSDP2MixedPrecisionSettings attribute) Reducer (class in modalities.running_env.fsdp.reducer) ReferenceConfig (class in modalities.config.config) referencing_keys (modalities.config.instantiation_models.TextGenerationInstantiationModel.TextGenerationSettings attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) RegexFilter (class in modalities.nn.model_initialization.parameter_name_filters) register_live_display() (modalities.logging_broker.subscriber_impl.progress_subscriber.RichProgressSubscriber class method) Registry (class in modalities.registry.registry) remove_key() (modalities.util.Aggregator method) remove_keys() (modalities.util.Aggregator method) repeat_kv() (in module modalities.conversion.gpt2.modeling_gpt2) repeat_kv_heads() (modalities.models.gpt2.gpt2_model.CausalSelfAttention class method) reset() (modalities.util.TimeRecorder method)

reset_parameters() (modalities.models.components.layer_norms.RMSLayerNorm method) reshard_after_forward (modalities.config.config.FSDP2WrappedModelConfig attribute) ResultItem (class in modalities.batch) ResultsSubscriberFactory (class in modalities.logging_broker.subscriber_impl.subscriber_factory) ResumableDistributedSampler (class in modalities.dataloader.samplers) ResumableDistributedSamplerConfig (class in modalities.config.config) RichProgressSubscriber (class in modalities.logging_broker.subscriber_impl.progress_subscriber) RichProgressSubscriberConfig (class in modalities.config.config) RichResultSubscriber (class in modalities.logging_broker.subscriber_impl.results_subscriber) RichResultSubscriberConfig (class in modalities.config.config) rms_norm (modalities.models.gpt2.gpt2_model.LayerNorms attribute) RMSLayerNorm (class in modalities.models.components.layer_norms) RMSLayerNormConfig (class in modalities.models.components.layer_norms) RotaryTransform (class in modalities.models.gpt2.gpt2_model) (modalities.models.gpt2.gpt2_model.QueryKeyValueTransformType attribute) rotate_half() (in module modalities.conversion.gpt2.modeling_gpt2) (modalities.models.gpt2.gpt2_model.RotaryTransform method) run() (modalities.dataloader.create_packed_data.PackedDataGenerator method) (modalities.gym.Gym method) (modalities.inference.text.inference_component.TextInferenceComponent method) run_checkpoint_instruction() (modalities.checkpointing.checkpoint_saving_execution.CheckpointSavingExecutionABC method) RUNNING (modalities.util.TimeRecorderStates attribute) RunningEnvError

S

sample_definition (modalities.dataloader.dataset.DummyDatasetConfig attribute) sample_key (modalities.config.config.GPT2LLMCollateFnConfig attribute) (modalities.config.config.MemMapDatasetConfig attribute) (modalities.config.config.PackedMemMapDatasetContinuousConfig attribute) (modalities.config.config.PackedMemMapDatasetMegatronConfig attribute) (modalities.dataloader.dataset.DummySampleConfig attribute) (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.models.huggingface.huggingface_model.HuggingFacePretrainedModelConfig attribute) (modalities.models.vision_transformer.vision_transformer_model.VisionTransformerConfig attribute) sample_keys (modalities.models.coca.collator.CoCaCollateFnConfig attribute) sample_shape (modalities.dataloader.dataset.DummySampleConfig attribute) sample_type (modalities.dataloader.dataset.DummySampleConfig attribute) sampler (modalities.config.config.BatchSamplerConfig attribute) samples (modalities.batch.DatasetBatch attribute) save_checkpoint() (modalities.checkpointing.checkpoint_saving.CheckpointSaving method) save_current (modalities.checkpointing.checkpoint_saving_instruction.CheckpointingInstruction attribute) SaveEveryKStepsCheckpointingStrategy (class in modalities.checkpointing.checkpoint_saving_strategies) SaveEveryKStepsCheckpointingStrategyConfig (class in modalities.config.config) SaveKMostRecentCheckpointsStrategy (class in modalities.checkpointing.checkpoint_saving_strategies) SaveKMostRecentCheckpointsStrategyConfig (class in modalities.config.config) SCALED (modalities.nn.model_initialization.parameter_name_filters.WeightInitTypes attribute) SCALED_EMBED (modalities.nn.model_initialization.parameter_name_filters.WeightInitTypes attribute) ScaledEmbedInitializationConfig (class in modalities.nn.model_initialization.initialization_routines) ScaledInitializationConfig (class in modalities.nn.model_initialization.initialization_routines) seed (modalities.config.config.DistributedSamplerConfig attribute) (modalities.config.config.ResumableDistributedSamplerConfig attribute) sentence_piece (modalities.utils.verify_tokenization_consistency.TokenizerTypes attribute) seq_length_dim (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig.RotaryTransformConfig attribute) sequence_length (modalities.config.config.GPT2MFUCalculatorConfig attribute) (modalities.config.config.PackedMemMapDatasetContinuousConfig attribute) (modalities.config.instantiation_models.StepProfile attribute) (modalities.config.instantiation_models.TextGenerationInstantiationModel.TextGenerationSettings attribute) (modalities.inference.text.config.TextInferenceComponentConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.utils.number_conversion.LocalNumBatchesFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumSamplesFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumStepsFromNumTokensConfig attribute) (modalities.utils.number_conversion.NumTokensFromNumStepsConfig attribute) (modalities.utils.number_conversion.NumTokensFromPackedMemMapDatasetContinuousConfig attribute) SequentialSamplerConfig (class in modalities.config.config) set_decoder() (modalities.conversion.gpt2.modeling_gpt2.GPT2ForCausalLM method)

set_input_embeddings() (modalities.conversion.gpt2.modeling_gpt2.GPT2ForCausalLM method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForQuestionAnswering method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForSequenceClassification method) (modalities.conversion.gpt2.modeling_gpt2.GPT2ForTokenClassification method) (modalities.conversion.gpt2.modeling_gpt2.GPT2Model method) set_output_embeddings() (modalities.conversion.gpt2.modeling_gpt2.GPT2ForCausalLM method) settings (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel attribute) (modalities.config.instantiation_models.TextGenerationInstantiationModel attribute) (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) sharding_strategy (modalities.config.config.FSDP1CheckpointLoadingConfig attribute) (modalities.config.config.FSDPWrappedModelConfig attribute) shuffle (modalities.config.config.DistributedSamplerConfig attribute) (modalities.config.config.ResumableDistributedSamplerConfig attribute) shuffle_file_chunks_in_place() (modalities.preprocessing.create_chunks.Chunking static method) shuffle_jsonl_data() (in module modalities.api) (modalities.preprocessing.shuffle_data.DataShuffler static method) shuffle_tokenized_data() (in module modalities.api) (modalities.preprocessing.shuffle_data.DataShuffler static method) SKIP (modalities.api.FileExistencePolicy attribute) skip_num_global_samples (modalities.config.config.ResumableDistributedSamplerConfig attribute) special_tokens (modalities.config.config.PreTrainedHFTokenizerConfig attribute) (modalities.tokenization.tokenizer_wrapper.PreTrainedHFTokenizer property) src_path (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel.PackedDatasetSettings attribute) start() (modalities.util.TimeRecorder method) start_factor (modalities.config.config.LinearLRSchedulerConfig attribute) state_dict() (modalities.checkpointing.stateful.app_state.AppState method) StatefulComponents (class in modalities.checkpointing.stateful.app_state) StateRetrieverIF (class in modalities.checkpointing.stateful.app_state) std (modalities.nn.model_initialization.composed_initialization.ComposedModelInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.PlainInitializationConfig attribute) (modalities.nn.model_initialization.initialization_routines.ScaledInitializationConfig attribute) step_profile (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) step_size (modalities.config.config.StepLRSchedulerConfig attribute) StepLRSchedulerConfig (class in modalities.config.config) StepProfile (class in modalities.config.instantiation_models) steps_per_epoch (modalities.config.config.OneCycleLRSchedulerConfig attribute) stop() (modalities.util.TimeRecorder method) STOPPED (modalities.util.TimeRecorderStates attribute) supports_gradient_checkpointing (modalities.conversion.gpt2.modeling_gpt2.GPT2PreTrainedModel attribute) SupportWeightInitModels (class in modalities.nn.model_initialization.parameter_name_filters) SwiGLU (class in modalities.models.model) SWIGLU (modalities.models.model.ActivationType attribute) sync_module_states (modalities.config.config.FSDPWrappedModelConfig attribute)

T

t_max (modalities.config.config.CosineAnnealingLRSchedulerConfig attribute) tag (modalities.loss_functions.Loss property) target_key (modalities.config.config.CLMCrossEntropyLossConfig attribute) (modalities.config.config.GPT2LLMCollateFnConfig attribute) target_keys (modalities.models.coca.collator.CoCaCollateFnConfig attribute) targets (modalities.batch.DatasetBatch attribute) (modalities.batch.InferenceResultBatch attribute) temperature (modalities.inference.text.config.TextInferenceComponentConfig attribute) tensor_parallel_degree (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) text_cls_prediction_key (modalities.models.coca.coca_model.CoCaConfig attribute) text_decoder_config (modalities.models.coca.coca_model.CoCaConfig attribute) text_embd_prediction_key (modalities.models.coca.coca_model.CoCaConfig attribute) text_inference_component (modalities.config.instantiation_models.TextGenerationInstantiationModel attribute) text_sample_key (modalities.models.coca.collator.CoCaCollateFnConfig attribute) text_target_key (modalities.models.coca.collator.CoCaCollateFnConfig attribute) TextDecoder (class in modalities.models.coca.text_decoder) TextDecoderConfig (class in modalities.models.coca.coca_model) TextGenerationInstantiationModel (class in modalities.config.instantiation_models) TextGenerationInstantiationModel.TextGenerationSettings (class in modalities.config.instantiation_models) TextInferenceComponent (class in modalities.inference.text.inference_component) TextInferenceComponentConfig (class in modalities.inference.text.config) three_phase (modalities.config.config.OneCycleLRSchedulerConfig attribute) throughput_metrics (modalities.batch.EvaluationResultBatch attribute) ThroughputAggregationKeys (class in modalities.trainer) TimeRecorder (class in modalities.util) TimeRecorderStateError TimeRecorderStates (class in modalities.util) to() (modalities.batch.DatasetBatch method) (modalities.batch.InferenceResultBatch method) (modalities.batch.TorchDeviceMixin method) to_cpu() (modalities.batch.InferenceResultBatch method) to_json_string() (modalities.models.huggingface_adapters.hf_adapter.HFModelAdapterConfig method) TOKEN_SIZE_DESCRIPTOR_LENGTH_IN_BYTES (modalities.dataloader.create_packed_data.EmbeddedStreamData attribute) (modalities.dataloader.dataset.PackedMemMapDatasetBase attribute) token_size_in_bytes (modalities.dataloader.dataset.PackedMemMapDatasetBase property) tokenize() (modalities.tokenization.tokenizer_wrapper.PreTrainedHFTokenizer method) (modalities.tokenization.tokenizer_wrapper.PreTrainedSPTokenizer method) (modalities.tokenization.tokenizer_wrapper.TokenizerWrapper method)

tokenizer (modalities.config.config.MemMapDatasetConfig attribute) (modalities.config.instantiation_models.PackedDatasetComponentsInstantiationModel attribute) (modalities.inference.text.config.TextInferenceComponentConfig attribute) tokenizer_model_file (modalities.config.config.PreTrainedSPTokenizerConfig attribute) TokenizerTypes (class in modalities.config.config) (class in modalities.utils.verify_tokenization_consistency) TokenizerWrapper (class in modalities.tokenization.tokenizer_wrapper) TorchCheckpointLoading (class in modalities.checkpointing.torch.torch_checkpoint_loading) TorchCheckpointLoadingConfig (class in modalities.config.config) TorchDeviceMixin (class in modalities.batch) total_iters (modalities.config.config.ConstantLRSchedulerConfig attribute) (modalities.config.config.LinearLRSchedulerConfig attribute) total_steps (modalities.config.config.OneCycleLRSchedulerConfig attribute) TP (modalities.running_env.fsdp.device_mesh.ParallelismDegrees attribute) TRAIN (modalities.logging_broker.messages.ExperimentStatus attribute) train() (modalities.trainer.Trainer method) train_dataloader (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) train_dataloader_tag (modalities.config.config.RichProgressSubscriberConfig attribute) train_dataset (modalities.config.instantiation_models.TrainingComponentsInstantiationModel attribute) Trainer (class in modalities.trainer) training_log_interval_in_steps (modalities.config.instantiation_models.Intervals attribute) training_progress (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) training_target (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) TrainingComponentsInstantiationModel (class in modalities.config.instantiation_models) TrainingComponentsInstantiationModel.Settings (class in modalities.config.instantiation_models) TrainingComponentsInstantiationModel.Settings.DCPWarmstartCheckpointPaths (class in modalities.config.instantiation_models) TrainingComponentsInstantiationModel.Settings.Paths (class in modalities.config.instantiation_models) TrainingComponentsInstantiationModel.Settings.Paths.Config (class in modalities.config.instantiation_models) TrainingComponentsInstantiationModel.Settings.WarmstartCheckpointPaths (class in modalities.config.instantiation_models) TrainingProgress (class in modalities.config.instantiation_models) (class in modalities.training.training_progress) TrainingReportGenerator (class in modalities.config.instantiation_models) TrainingTarget (class in modalities.config.instantiation_models) transfer_model_code() (in module modalities.conversion.gpt2.conversion_code) TransformerBlock (class in modalities.models.coca.multi_modal_decoder) TransformerMLP (class in modalities.models.gpt2.gpt2_model) truncation (modalities.config.config.PreTrainedHFTokenizerConfig attribute) type_converter_for_torch (modalities.dataloader.dataset.PackedMemMapDatasetBase attribute) type_hint (modalities.models.gpt2.gpt2_model.AttentionConfig.QueryKeyValueTransformConfig attribute)

U

use_meta_device (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute)

use_weight_tying (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute)

V

validate_dp_mesh_existence() (modalities.config.config.FSDP2WrappedModelConfig method) validate_mixed_precision_settings() (modalities.config.config.FSDP2WrappedModelConfig method) validate_sizes() (modalities.models.gpt2.gpt2_model.GPT2LLMConfig method) value (modalities.batch.ResultItem attribute) variant_key (modalities.registry.components.ComponentEntity attribute) verify_tokenization_consistency() (in module modalities.utils.verify_tokenization_consistency) vision_cls_prediction_key (modalities.models.coca.coca_model.CoCaConfig attribute) vision_embd_prediction_key (modalities.models.coca.coca_model.CoCaConfig attribute)

vision_encoder_config (modalities.models.coca.coca_model.CoCaConfig attribute) VisionTransformer (class in modalities.models.vision_transformer.vision_transformer_model) VisionTransformerBlock (class in modalities.models.vision_transformer.vision_transformer_model) VisionTransformerConfig (class in modalities.models.vision_transformer.vision_transformer_model) vocab_size (modalities.models.coca.coca_model.TextDecoderConfig attribute) (modalities.models.gpt2.gpt2_model.GPT2LLMConfig attribute) (modalities.tokenization.tokenizer_wrapper.PreTrainedHFTokenizer property) (modalities.tokenization.tokenizer_wrapper.PreTrainedSPTokenizer property) (modalities.tokenization.tokenizer_wrapper.TokenizerWrapper property)

W

WandBEvaluationResultSubscriber (class in modalities.logging_broker.subscriber_impl.results_subscriber) WandBEvaluationResultSubscriberConfig (class in modalities.config.config) WandbMode (class in modalities.config.config) warmstart_checkpoint_paths (modalities.config.instantiation_models.TrainingComponentsInstantiationModel.Settings attribute) warn_rank_0() (in module modalities.util) weight_decay (modalities.config.config.AdamOptimizerConfig attribute) (modalities.config.config.AdamWOptimizerConfig attribute) weight_decay_groups (modalities.models.model.NNModel property) weight_decay_groups_excluded (modalities.config.config.AdamOptimizerConfig attribute) (modalities.config.config.AdamWOptimizerConfig attribute) weight_init_type (modalities.nn.model_initialization.composed_initialization.ComposedModelInitializationConfig attribute)

WeightInitializedModelConfig (class in modalities.config.config) WeightInitTypes (class in modalities.nn.model_initialization.parameter_name_filters) weights (modalities.nn.model_initialization.parameter_name_filters.RegexFilter attribute) world_size (modalities.config.config.GPT2MFUCalculatorConfig attribute) (modalities.config.instantiation_models.CudaEnvSettings attribute) (modalities.running_env.fsdp.device_mesh.DeviceMeshConfig attribute) wrapped_model (modalities.config.config.AdamOptimizerConfig attribute) (modalities.config.config.AdamWOptimizerConfig attribute) (modalities.config.config.FSDP1CheckpointedOptimizerConfig attribute) (modalities.config.config.GPT2MFUCalculatorConfig attribute) (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.FSDPDummyGradientClipperConfig attribute) (modalities.training.gradient_clipping.fsdp_gradient_clipper_config.FSDPGradientClipperConfig attribute)