B1a504a7 E1fc 424f 99e4 366a07357bfa

Source document

# Load pre-trained model and tokenizer model = AutoModel.from_pretrained('distilbert-base-uncased') tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased') # Define a function to calculate embedding dimensions def calculate_embedding_dimensions(input_ids, attention_mask): try: # Calculate embedding dimensions outputs = model(input_ids, attention_mask=attention_mask) embedding_dimensions = outputs[0].shape return embedding_dimensions except RuntimeError as e: if "size mismatch" in str(e): raise ValueError("EmbeddingDimensionError: Size mismatch in input dimensions.") from e else: raise e # Define a function to preprocess input data def preprocess_input_data(texts): inputs = tokenizer(texts, padding=True, truncation=True, return_tensors='pt') return inputs['input_ids'], inputs['attention_mask'] # Test the function texts = ["This is a test sentence.", "Another test sentence."] input_ids, attention_mask = preprocess_input_data(texts) try: embedding_dimensions = calculate_embedding_dimensions(input_ids, attention_mask)