CoolFace
Datasetpublic

feyninc/CVPR_2024_Papers

def remove_ref_tags_except_image_from_sample(sample): """ cleanup the deepseek exrtra tags """ lines = sample["text"].split('\n') filtered_lines = [] for line in lines: # Check if line contains any <|ref|> tag if '<|ref|>' in line: # Keep the line only if it contains <|ref|>image<|/ref|> if '<|ref|>image<|/ref|>' in line: filtered_lines.append(line) else: # Keep lines without <|ref|> tags… See the full description on the dataset page: https://huggingface.co/datasets/feyninc/CVPR_2024_Papers.

sourceHugging Faceupdated 10mo agoView on Hugging Face
0likes30downloads
Dataset Card
python
def remove_ref_tags_except_image_from_sample(sample):
    """
    cleanup the deepseek exrtra tags
    """
    lines = sample["text"].split('\n')
    filtered_lines = []
    
    for line in lines:
        # Check if line contains any <|ref|> tag
        if '<|ref|>' in line:
            # Keep the line only if it contains <|ref|>image<|/ref|>
            if '<|ref|>image<|/ref|>' in line:
                filtered_lines.append(line)
        else:
            # Keep lines without <|ref|> tags
            filtered_lines.append(line)
    
    out = '\n'.join(filtered_lines)
    return {'text':out.strip()}