B
��^� � @ s� d dl Zd dlm mZ d dlZd dlZd dlm Z d dl
Z
d dl m Z d dl
mZ d dlmZ d dlmZ d dlZd dlmZ G dd � d �ZdS )
� N)�Path)�get_from_cache)�
BertTokenizer)�PyBertTokenizer)� BertForSequenceClassification)�
default_timerc @ s\ e Zd Zdd� Zdd� Zdd� Zdd� Zd d
� Zd d � Zd
d� Z dd� Z
dd� Z dd� Z dS )�TestBenchmarkBertc s t j�� � _tt�� �� _tj dd� jd�� _
t t � j
j
d d ��� _tj ddd��� � _� jrl� j�� dgd � _� fd d
�� jD �}� fd d
�|D �}� fd d
�|D �}t jd
d
� |D �t jd�}� jr�|�� }t �� � � �|�d �� �� }W d Q R X d S )Nzbert-base-uncasedT)�
do_lower_case� cache_dir�
vocab_fileF)Zoutput_attentionsu� For instance, on the planet Earth, man had always assumed that he was more intelligent than dolphins because he had achieved so much—the wheel, New York, wars and so on—whilst all the dolphins had ever done was muck about in the water having a good time. But conversely, the dolphins had always believed that they were far more intelligent than man—for precisely the same reasons.�@ c s g | ]}� j �|��qS � )�base_tokenizer�tokenize)�.0�sentence)�selfr
�DE:\Coding\backup-rust\rust-transformers\tests\test_benchmark_bert.py�
<listcomp>, s z1TestBenchmarkBert.setup_class.<locals>.<listcomp>c s g | ]}� j �|��qS r
)r �convert_tokens_to_ids)r �tokens)r r
r r - s c s g | ]}� j j|d ddd��qS )NT� )�add_special_tokens�
max_length)r �prepare_for_model)r �input)r r
r r . s c S s g | ] }|d �qS )� input_idsr
)r �fr
r
r r 1 s )�dtyper )�torch�cudaZ is_available�use_gpur �tempfile�mkdtemp�test_dirr �from_pretrainedr r r �pretrained_vocab_files_map�rust_tokenizerr �eval�model�
sentence_list�tensor�long�no_grad�cpu�numpy)r � tokens_list�features�
all_input_ids�_r
)r r � setup_class s(
z TestBenchmarkBert.setup_classc C s t jdd| jd�| _d S )Nzbert-base-uncasedT)r r
)r r% r$ r )r r
r
r �setup_base_tokenizer9 s z&TestBenchmarkBert.setup_base_tokenizerc C s t t| jjd d ��| _d S )Nr zbert-base-uncased)r r r r& r' )r r
r
r �setup_rust_tokenizer= s z&TestBenchmarkBert.setup_rust_tokenizerc s� � fdd�� j D �}� fdd�|D �}� fdd�|D �}tjdd� |D �tjd�}� jr`|�� }t�� � � �|�d �� � � }W d Q R X |S )Nc s g | ]}� j �|��qS r
)r r )r r )r r
r r B s z4TestBenchmarkBert.baseline_batch.<locals>.<listcomp>c s g | ]}� j �|��qS r
)r r )r r )r r
r r C s c s g | ]}� j j|d ddd��qS )NTr )r r )r r )r r )r r
r r D s c S s g | ] }|d �qS )r r
)r r r
r
r r H s )r r )
r* r r+ r, r! r r- r) r. r/ )r r0 r1 r2 �outputr
)r r �baseline_batchA s
z TestBenchmarkBert.baseline_batchc sj � fdd�� j D �}tjdd� |D �tjd�}� jr<|�� }t�� � � �|�d �� � � }W d Q R X |S )Nc s g | ]}� j j|d ddd��qS )r �
longest_firstr )�max_len�truncation_strategy�stride)r' �encode)r r )r r
r r P s z@TestBenchmarkBert.rust_batch_single_threaded.<locals>.<listcomp>c S s g | ]
}|j �qS r
)� token_ids)r r r
r
r r T s )r r )
r* r r+ r, r! r r- r) r. r/ )r r1 r2 r7 r
)r r �rust_batch_single_threadedO s
z,TestBenchmarkBert.rust_batch_single_threadedc C sl | j j| jdddd�}tjdd� |D �tjd�}| jr>|�� }t�� � | � |�d �
� � � }W d Q R X |S )Nr r9 r )r: r; r< c S s g | ]
}|j �qS r
)r> )r r r
r
r r ` s z?TestBenchmarkBert.rust_batch_multi_threaded.<locals>.<listcomp>)r ) r' Z encode_listr* r r+ r, r! r r- r) r. r/ )r r1 r2 r7 r
r
r �rust_batch_multi_threaded[ s
z+TestBenchmarkBert.rust_batch_multi_threadedc s� g }x>t d�D ]2}| �� t� }| �� t� }|�|| d � qW t|�t|� � t�t� fdd�|D ���t|�d }t d� d�d|d��� d S ) N�
i� c s g | ]}|� d �qS )� r
)r �value)�meanr
r r p s z8TestBenchmarkBert.test_bert_baseline.<locals>.<listcomp>� zbaseline - mean: z.2fz , std. dev: )
�ranger5 �timerr8 �append�sum�len�math�sqrt�print)r �values�i�t0�t1�std_devr
)rD r �test_bert_baselineg s (z$TestBenchmarkBert.test_bert_baselinec s� g }x>t d�D ]2}| �� t� }| �� t� }|�|| d � qW t|�t|� � t�t� fdd�|D ���t|�d }t d� d�d|d��� d S ) NrA i� c s g | ]}|� d �qS )rB r
)r rC )rD r
r r | s zDTestBenchmarkBert.test_bert_rust_single_threaded.<locals>.<listcomp>rE zrust single thread - mean: z.2fz , std. dev: )
rF r6 rG r? rH rI rJ rK rL rM )r rN rO rP rQ rR r
)rD r � test_bert_rust_single_threadeds s (z0TestBenchmarkBert.test_bert_rust_single_threadedc s� g }x>t d�D ]2}| �� t� }| �� t� }|�|| d � qW t|�t|� � t�t� fdd�|D ���t|�d }t d� d�d|d��� d S ) NrA i� c s g | ]}|� d �qS )rB r
)r rC )rD r
r r � s zCTestBenchmarkBert.test_bert_rust_multi_threaded.<locals>.<listcomp>rE z rust multi threaded - mean: z.2fz , std. dev: )
rF r6 rG r@ rH rI rJ rK rL rM )r rN rO rP rQ rR r
)rD r � test_bert_rust_multi_threaded s (z/TestBenchmarkBert.test_bert_rust_multi_threadedc C s( d | _ d | _d | _t�� tj�� d S )N)r) r r' �gc�collectr r Z empty_cache)r r
r
r �teardown_class� s
z TestBenchmarkBert.teardown_classN)
�__name__�
__module__� __qualname__r4 r5 r6 r8 r? r@ rS rT rU rX r
r
r
r r s r )�builtins� @py_builtins�_pytest.assertion.rewrite� assertion�rewrite�
@pytest_arrK r" �pathlibr rV Ztransformers.file_utilsr Z transformers.tokenization_bertr Zrust_transformersr Ztransformers.modeling_bertr r �timeitr rG r r
r
r
r �<module>
s