maxtext.input_pipeline package

Subpackages

• maxtext.input_pipeline.packing package
  • Submodules
    • maxtext.input_pipeline.packing.prefill_packing module
    • maxtext.input_pipeline.packing.sequence_packing module
      • pack_dataset()
• maxtext.input_pipeline.protos package
  • Submodules
    • maxtext.input_pipeline.protos.example_pb2 module
      • Example
      • SequenceExample
    • maxtext.input_pipeline.protos.feature_pb2 module
      • BytesList
      • Feature
      • FeatureList
      • FeatureLists
      • Features
      • FloatList
      • Int64List

Submodules

• maxtext.input_pipeline.data_processing_utils module
  • parse_and_keep_features()
  • get_tokenizer_and_pad_id()
  • validate_and_configure_sft_columns()
  • get_local_batch_size()
  • format_and_batch()
  • apply_multiprocessing_and_prefetch()
• maxtext.input_pipeline.distillation_data_processing module
  • InputRequest
    • InputRequest.prompt
    • InputRequest.prompt_token_ids
    • InputRequest.actual_completion
    • InputRequest.max_output_tokens
  • map_to_prompt_completion()
  • extract_content()
  • process_dataset()
  • load_dataset()
  • filter_dataset()
• maxtext.input_pipeline.grain_data_processing module
  • find_data_files()
  • get_datasets()
  • pretrain_preprocessing_pipeline()
  • dpo_preprocessing_pipeline()
  • sft_preprocessing_pipeline()
  • make_grain_train_iterator()
  • make_grain_eval_iterator()
• maxtext.input_pipeline.grain_tokenizer module
  • TokenizerTransformBase
    • TokenizerTransformBase.feature_names
    • TokenizerTransformBase.sequence_length
    • TokenizerTransformBase.tokenizer
  • TokenizeAndTrim
    • TokenizeAndTrim.map()
  • TokenizeAndChunk
    • TokenizeAndChunk.max_fan_out
    • TokenizeAndChunk.flat_map()
• maxtext.input_pipeline.hf_data_processing module
  • vision_sft_preprocessing_pipeline()
  • preprocessing_pipeline()
  • make_hf_train_iterator()
  • make_hf_eval_iterator()
• maxtext.input_pipeline.input_pipeline_interface module
  • get_process_loading_real_data()
  • create_process_specific_iterator()
  • create_data_iterator()
• maxtext.input_pipeline.input_pipeline_utils module
  • normalize_features()
  • get_tokenizer()
  • truncate_to_max_allowable_length()
  • shift_data_by_truncation()
  • add_segmentation_and_position()
  • TokenizeOp()
  • reformat_prompt()
  • reformat_response()
  • merge_image_columns()
  • pre_process_image_sft()
  • prepare_text_for_image_fusion()
  • combine_columns()
  • is_conversational()
  • extract_token_ids()
  • verify_chat_template_generation_prompt_logic()
  • apply_chat_template()
  • tokenization()
  • SFTPromptMasking
    • SFTPromptMasking.map()
  • SFTPromptMaskingVision
    • SFTPromptMaskingVision.map()
  • HFNormalizeFeatures
    • HFNormalizeFeatures.map()
  • HFDataSource
  • GCSTFRecordIterDataset
  • make_tfrecord_iter_dataset()
  • ParseFeatures
    • ParseFeatures.map()
  • NormalizeFeatures
    • NormalizeFeatures.map()
  • KeepFeatures
    • KeepFeatures.map()
  • Rekey
    • Rekey.map()
  • ReformatPacking
    • ReformatPacking.map()
  • PadOrTrimToMaxLength
    • PadOrTrimToMaxLength.map()
  • ExtractImagesAndMasks
    • ExtractImagesAndMasks.map()
  • FoldImagesIntoBatch
    • FoldImagesIntoBatch.model_name
    • FoldImagesIntoBatch.map()
  • shift_right()
  • shift_left()
  • shift_and_refine()
  • ShiftData
    • ShiftData.map()
  • ComputeQwen3OmniPositions
    • ComputeQwen3OmniPositions.map()
• maxtext.input_pipeline.instruction_data_processing module
  • load_data_template_from_file()
  • load_chat_template_from_file()
  • get_template_placeholders()
  • extract_reasoning_and_answer()
  • math_qa_formatting()
  • load_formatter()
  • convert_to_conversational_format()
• maxtext.input_pipeline.multihost_dataloading module
  • MultiHostDataLoadIterator
    • MultiHostDataLoadIterator.reset()
  • RemoteIteratorWrapper
    • RemoteIteratorWrapper.reset()
    • RemoteIteratorWrapper.save_state()
    • RemoteIteratorWrapper.restore_state()
• maxtext.input_pipeline.olmo_data module
  • OlmoNpyFileEntry
    • OlmoNpyFileEntry.path
    • OlmoNpyFileEntry.label
    • OlmoNpyFileEntry.n_tokens
    • OlmoNpyFileEntry.n_instances
    • OlmoNpyFileEntry.instance_offset
  • OlmoNpyIndex
    • OlmoNpyIndex.format_version
    • OlmoNpyIndex.sequence_length
    • OlmoNpyIndex.dtype
    • OlmoNpyIndex.tokenizer
    • OlmoNpyIndex.files
    • OlmoNpyIndex.total_instances
    • OlmoNpyIndex.total_tokens
    • OlmoNpyIndex.fingerprint
    • OlmoNpyIndex.to_json_dict()
    • OlmoNpyIndex.save()
  • load_index()
  • global_to_local()
  • compute_fingerprint()
  • parse_npy_header()
  • read_npy_header_from_path()
  • read_raw_metadata_from_path()
  • has_npy_magic()
  • build_index()
  • RepetitionTuple
    • RepetitionTuple.start
    • RepetitionTuple.end
    • RepetitionTuple.period
    • RepetitionTuple.times
  • find_periodic_sequences()
  • is_clean_instance()
• maxtext.input_pipeline.olmo_data_grain module
  • OlmoNpyDataSource
  • OlmoIndexSampler
    • OlmoIndexSampler.num_instances
    • OlmoIndexSampler.num_local_instances_per_epoch
    • OlmoIndexSampler.shuffled_global_indices()
    • OlmoIndexSampler.shard_indices()
  • NgramFilterTransform
    • NgramFilterTransform.map()
  • ShiftToInputsTargets
    • ShiftToInputsTargets.map()
  • make_olmo_grain_data_loader()
• maxtext.input_pipeline.olmo_grain_data_processing module
  • make_olmo_grain_train_iterator()
  • make_olmo_grain_eval_iterator()
• maxtext.input_pipeline.synthetic_data_processing module
  • SyntheticDataIterator
    • SyntheticDataIterator.data_generator
    • SyntheticDataIterator.reset()
    • SyntheticDataIterator.raw_generate_synthetic_data()
  • PlaceHolderDataIterator
    • PlaceHolderDataIterator.reset()
    • PlaceHolderDataIterator.get_place_holder_synthetic_data()
• maxtext.input_pipeline.tfds_data_processing module
  • get_datasets()
  • preprocessing_pipeline()
  • make_tfds_train_iterator()
  • make_tfds_eval_iterator()
• maxtext.input_pipeline.tfds_data_processing_c4_mlperf module
  • rekey()
  • reduce_concat_tokens()
  • split_tokens()
  • split_tokens_to_targets_length()
  • get_dataset()
  • format_fn()
  • preprocess_train_dataset()
  • preprocess_eval_dataset()
  • make_c4_mlperf_train_iterator()
  • make_c4_mlperf_eval_iterator()
• maxtext.input_pipeline.tokenizer module
  • TikTokenTokenizer
    • TikTokenTokenizer.num_reserved_special_tokens
    • TikTokenTokenizer.pat_str
    • TikTokenTokenizer.special_tokens
    • TikTokenTokenizer.encode()
    • TikTokenTokenizer.decode()
  • SentencePieceTokenizer
    • SentencePieceTokenizer.encode()
    • SentencePieceTokenizer.decode()
  • HFTokenizer
    • HFTokenizer.encode()
    • HFTokenizer.decode()
  • build_tokenizer()