trfms-integration (#3)

- trfs integration (31083d9ebf98f72055597970738f24a6644d90cd)
- readme update (1d5178a0d33d436744976db4448f73be16d0bfdf)
- readme update (e421a862718ad77e23463222b0644b8086fcf2ef)
- readme update (0090acf9edb289ef455a8c284af6325f4608b3f2)
- readme update (76d150f3f15da56e290b47660b4e9daaea0e83cb)


Co-authored-by: Eustache Le Bihan <[email protected]>

README.md

@@ -18,6 +18,7 @@ datasets:
 - MLCommons/peoples_speech
 thumbnail: null
 tags:
+- transformers
 - automatic-speech-recognition
 - speech
 - audio
@@ -191,7 +192,7 @@ pip install nemo_toolkit['all']
 
 ## How to Use this Model
 
-The model is available for use in the NeMo toolkit [3], and can be used as a pre-trained checkpoint for inference or for fine-tuning on another dataset.
+The model is available for use in the NeMo toolkit [3], and can be used as a pre-trained checkpoint for inference or for fine-tuning on another dataset. Moreover, you can now run Parakeet CTC natively with [Transformers](https://github.com/huggingface/transformers) 🤗.
 
 ### Automatically instantiate the model
 
@@ -200,7 +201,7 @@ import nemo.collections.asr as nemo_asr
 asr_model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(model_name="nvidia/parakeet-ctc-0.6b")
 ```
 
-### Transcribing using Python
+### Transcribing using NeMo
 First, let's get a sample
 ```
 wget https://dldata-public.s3.us-east-2.amazonaws.com/2086-149220-0033.wav
@@ -210,6 +211,79 @@ Then simply do:
 asr_model.transcribe(['2086-149220-0033.wav'])
 ```
 
+### Transcribing using [Transformers](https://github.com/huggingface/transformers) 🤗 
+
+Make sure to install `transformers` from source.
+
+```bash
+pip install git+https://github.com/huggingface/transformers
+```
+
+<details>
+  <summary>➡️ Pipeline usage</summary>
+
+```python
+from transformers import pipeline
+
+pipe = pipeline("automatic-speech-recognition", model="nvidia/parakeet-ctc-0.6b")
+out = pipe("https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/bcn_weather.mp3")
+print(out)
+```
+</details>
+
+<details>
+  <summary>➡️ AutoModel</summary>
+
+```python
+from transformers import AutoModelForCTC, AutoProcessor
+from datasets import load_dataset, Audio
+import torch
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+processor = AutoProcessor.from_pretrained("nvidia/parakeet-ctc-0.6b")
+model = AutoModelForCTC.from_pretrained("nvidia/parakeet-ctc-0.6b", dtype="auto", device_map=device)
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))
+speech_samples = [el['array'] for el in ds["audio"][:5]]
+
+inputs = processor(speech_samples, sampling_rate=processor.feature_extractor.sampling_rate)
+inputs.to(model.device, dtype=model.dtype)
+outputs = model.generate(**inputs)
+print(processor.batch_decode(outputs))
+```
+</details>
+
+<details>
+  <summary>➡️ Training</summary>
+
+```python
+from transformers import AutoModelForCTC, AutoProcessor
+from datasets import load_dataset, Audio
+import torch
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+processor = AutoProcessor.from_pretrained("nvidia/parakeet-ctc-0.6b")
+model = AutoModelForCTC.from_pretrained("nvidia/parakeet-ctc-0.6b", dtype="auto", device_map=device)
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))
+speech_samples = [el['array'] for el in ds["audio"][:5]]
+text_samples = [el for el in ds["text"][:5]]
+
+# passing `text` to the processor will prepare inputs' `labels` key
+inputs = processor(audio=speech_samples, text=text_samples, sampling_rate=processor.feature_extractor.sampling_rate)
+inputs.to(device, dtype=model.dtype)
+
+outputs = model(**inputs)
+outputs.loss.backward()
+```
+</details>
+
+For more details about usage, the refer to [Transformers' documentation](https://huggingface.co/docs/transformers/en/index).
+
 ### Transcribing many audio files
 
 ```shell

config.json

@@ -0,0 +1,37 @@
+{
+  "architectures": [
+    "ParakeetForCTC"
+  ],
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": true,
+  "dtype": "bfloat16",
+  "encoder_config": {
+    "activation_dropout": 0.1,
+    "attention_bias": true,
+    "attention_dropout": 0.1,
+    "conv_kernel_size": 9,
+    "dropout": 0.1,
+    "dropout_positions": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layerdrop": 0.1,
+    "max_position_embeddings": 5000,
+    "model_type": "parakeet_encoder",
+    "num_attention_heads": 8,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 8,
+    "num_mel_bins": 80,
+    "scale_input": true,
+    "subsampling_conv_channels": 256,
+    "subsampling_conv_kernel_size": 3,
+    "subsampling_conv_stride": 2,
+    "subsampling_factor": 8
+  },
+  "initializer_range": 0.02,
+  "model_type": "parakeet_ctc",
+  "pad_token_id": 1024,
+  "transformers_version": "4.57.0.dev0",
+  "vocab_size": 1025
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1f5b1d2bc66d4a812a1c8ea48a9aa8678f863e1bd3ecbec3c75abd720897fda
+size 2435506188

preprocessor_config.json

@@ -0,0 +1,13 @@
+{
+  "feature_extractor_type": "ParakeetFeatureExtractor",
+  "feature_size": 80,
+  "hop_length": 160,
+  "n_fft": 512,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "preemphasis": 0.97,
+  "processor_class": "ParakeetProcessor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000,
+  "win_length": 400
+}

special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,27 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1024": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": false,
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "ParakeetProcessor",
+  "tokenizer_class": "ParakeetTokenizerFast",
+  "unk_token": "<unk>"
+}