predict_submit_seresnext101

From: https://www.kaggle.com/blondinka/predict-submit-seresnext101

Author: Blonde

Score: 0.631

In [1]:
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 
import argparse
from pathlib import Path
from typing import Callable, List
import numpy as np
import pandas as pd
import tqdm
from multiprocessing.pool import ThreadPool

import torch
from torch import nn, cuda
from torch.nn import functional as F
import torchvision.models as M
from torch.utils.data import DataLoader
from torch.utils.data import Dataset 
    
import cv2
from PIL import Image
from torchvision.transforms import (
    ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop,
    RandomHorizontalFlip, RandomGrayscale)
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))
['pytorch-model-zoo', 'imet-2019-fgvc6', 'seresnext101-folds', 'densenet201-5folds']
In [2]:
ON_KAGGLE = True
N_CLASSES = 1103
DATA_ROOT = Path('../input/imet-2019-fgvc6' if ON_KAGGLE else '../data')
RUN_ROOT = '../input/seresnext101-folds/' if ON_KAGGLE else '../data/results/'
use_sample = False
use_cuda = cuda.is_available()
SIZE = 352

train_root = DATA_ROOT / 'train'
test_root = DATA_ROOT / 'test'
In [3]:
import os
print('Files present in this directory', os.listdir(RUN_ROOT))
print('Files present in this directory', os.listdir(DATA_ROOT))
Files present in this directory ['best-model_fold0.pt', 'best-metric_fold4.pt', 'best-model_fold11.pt', 'best-metric_fold2.pt', 'best-metric_fold1.pt', 'best-model_fold4.pt', 'best-metric_fold3.pt', 'best-metric_fold0.pt', 'best-metric_fold11.pt']
Files present in this directory ['test', 'train', 'train.csv', 'labels.csv', 'sample_submission.csv']
In [4]:
"""

SeResnet models

https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py
"""

from collections import OrderedDict
import math
import torch.nn as nn
from torch.utils import model_zoo

class SEModule(nn.Module): 

    def __init__(self, channels, reduction):
        super(SEModule, self).__init__()
        self.avg_pool = nn.AdaptiveAvgPool2d(1)
        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
                             padding=0)
        self.relu = nn.ReLU(inplace=True)
        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
                             padding=0)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        module_input = x
        x = self.avg_pool(x)
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        x = self.sigmoid(x)
        return module_input * x


class Bottleneck(nn.Module):
    """
    Base class for bottlenecks that implements `forward()` method.
    """
    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out = self.se_module(out) + residual
        out = self.relu(out)

        return out


class SEBottleneck(Bottleneck):
    """
    Bottleneck for SENet154.
    """
    expansion = 4

    def __init__(self, inplanes, planes, groups, reduction, stride=1,
                 downsample=None):
        super(SEBottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes * 2)
        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
                               stride=stride, padding=1, groups=groups,
                               bias=False)
        self.bn2 = nn.BatchNorm2d(planes * 4)
        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
                               bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.se_module = SEModule(planes * 4, reduction=reduction)
        self.downsample = downsample
        self.stride = stride


class SEResNetBottleneck(Bottleneck):
    """
    ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe
    implementation and uses `stride=stride` in `conv1` and not in `conv2`
    (the latter is used in the torchvision implementation of ResNet).
    """
    expansion = 4

    def __init__(self, inplanes, planes, groups, reduction, stride=1,
                 downsample=None):
        super(SEResNetBottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
                               stride=stride)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
                               groups=groups, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.se_module = SEModule(planes * 4, reduction=reduction)
        self.downsample = downsample
        self.stride = stride


class SEResNeXtBottleneck(Bottleneck):
    """
    ResNeXt bottleneck type C with a Squeeze-and-Excitation module.
    """
    expansion = 4

    def __init__(self, inplanes, planes, groups, reduction, stride=1,
                 downsample=None, base_width=4):
        super(SEResNeXtBottleneck, self).__init__()
        width = math.floor(planes * (base_width / 64)) * groups
        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
                               stride=1)
        self.bn1 = nn.BatchNorm2d(width)
        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
                               padding=1, groups=groups, bias=False)
        self.bn2 = nn.BatchNorm2d(width)
        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.se_module = SEModule(planes * 4, reduction=reduction)
        self.downsample = downsample
        self.stride = stride


class SENet(nn.Module):

    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
                 inplanes=128, input_3x3=True, downsample_kernel_size=3,
                 downsample_padding=1, num_classes=1000):
        """
        Parameters
        ----------
        block (nn.Module): Bottleneck class.
            - For SENet154: SEBottleneck
            - For SE-ResNet models: SEResNetBottleneck
            - For SE-ResNeXt models:  SEResNeXtBottleneck
        layers (list of ints): Number of residual blocks for 4 layers of the
            network (layer1...layer4).
        groups (int): Number of groups for the 3x3 convolution in each
            bottleneck block.
            - For SENet154: 64
            - For SE-ResNet models: 1
            - For SE-ResNeXt models:  32
        reduction (int): Reduction ratio for Squeeze-and-Excitation modules.
            - For all models: 16
        dropout_p (float or None): Drop probability for the Dropout layer.
            If `None` the Dropout layer is not used.
            - For SENet154: 0.2
            - For SE-ResNet models: None
            - For SE-ResNeXt models: None
        inplanes (int):  Number of input channels for layer1.
            - For SENet154: 128
            - For SE-ResNet models: 64
            - For SE-ResNeXt models: 64
        input_3x3 (bool): If `True`, use three 3x3 convolutions instead of
            a single 7x7 convolution in layer0.
            - For SENet154: True
            - For SE-ResNet models: False
            - For SE-ResNeXt models: False
        downsample_kernel_size (int): Kernel size for downsampling convolutions
            in layer2, layer3 and layer4.
            - For SENet154: 3
            - For SE-ResNet models: 1
            - For SE-ResNeXt models: 1
        downsample_padding (int): Padding for downsampling convolutions in
            layer2, layer3 and layer4.
            - For SENet154: 1
            - For SE-ResNet models: 0
            - For SE-ResNeXt models: 0
        num_classes (int): Number of outputs in `last_linear` layer.
            - For all models: 1000
        """
        super(SENet, self).__init__()
        self.inplanes = inplanes
        if input_3x3:
            layer0_modules = [
                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
                                    bias=False)),
                ('bn1', nn.BatchNorm2d(64)),
                ('relu1', nn.ReLU(inplace=True)),
                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
                                    bias=False)),
                ('bn2', nn.BatchNorm2d(64)),
                ('relu2', nn.ReLU(inplace=True)),
                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
                                    bias=False)),
                ('bn3', nn.BatchNorm2d(inplanes)),
                ('relu3', nn.ReLU(inplace=True)),
            ]
        else:
            layer0_modules = [
                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
                                    padding=3, bias=False)),
                ('bn1', nn.BatchNorm2d(inplanes)),
                ('relu1', nn.ReLU(inplace=True)),
            ]
        # To preserve compatibility with Caffe weights `ceil_mode=True`
        # is used instead of `padding=1`.
        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,
                                                    ceil_mode=True)))
        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
        self.layer1 = self._make_layer(
            block,
            planes=64,
            blocks=layers[0],
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=1,
            downsample_padding=0
        )
        self.layer2 = self._make_layer(
            block,
            planes=128,
            blocks=layers[1],
            stride=2,
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=downsample_kernel_size,
            downsample_padding=downsample_padding
        )
        self.layer3 = self._make_layer(
            block,
            planes=256,
            blocks=layers[2],
            stride=2,
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=downsample_kernel_size,
            downsample_padding=downsample_padding
        )
        self.layer4 = self._make_layer(
            block,
            planes=512,
            blocks=layers[3],
            stride=2,
            groups=groups,
            reduction=reduction,
            downsample_kernel_size=downsample_kernel_size,
            downsample_padding=downsample_padding
        )
        #self.avg_pool = nn.AvgPool2d(7, stride=1) 
        self.avg_pool = nn.AdaptiveAvgPool2d(1)

        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None
        self.last_linear = nn.Linear(512 * block.expansion, num_classes)

    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
                    downsample_kernel_size=1, downsample_padding=0):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=downsample_kernel_size, stride=stride,
                          padding=downsample_padding, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, groups, reduction, stride,
                            downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes, groups, reduction))

        return nn.Sequential(*layers)

    def features(self, x):
        x = self.layer0(x)
        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        return x

    def logits(self, x):
        x = self.avg_pool(x)
        if self.dropout is not None:
            x = self.dropout(x)
        x = x.view(x.size(0), -1)
        x = self.last_linear(x)
        return x

    def forward(self, x):
        x = self.features(x)
        x = self.logits(x)
        return x


def initialize_pretrained_model(model, num_classes, settings):
    assert num_classes == settings['num_classes'], \
        'num_classes should be {}, but is {}'.format(
            settings['num_classes'], num_classes)
    model.load_state_dict(model_zoo.load_url(settings['url']))
    model.input_space = settings['input_space']
    model.input_size = settings['input_size']
    model.input_range = settings['input_range']
    model.mean = settings['mean']
    model.std = settings['std']

def se_resnext101(num_classes=1000, pretrained=None):
    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
                  dropout_p=0.5, inplanes=64, input_3x3=False,
                  downsample_kernel_size=1, downsample_padding=0,
                  num_classes=num_classes)    
    return model
In [5]:
model = se_resnext101()
model.last_linear = nn.Linear(model.last_linear.in_features, N_CLASSES)

all_params = list(model.parameters())
use_cuda = cuda.is_available()
if use_cuda:
    model = model.cuda()
In [6]:
def load_model(model, root: str, fold: int, use_cuda: bool):
    """Loads model checkpoints
       Choose evaluation mode
    """
    best_model_path = root + 'best-metric_fold'+str(fold)+'.pt'
    if use_cuda:
        state_dict = torch.load(best_model_path)
    else:
        state_dict = torch.load(best_model_path, map_location='cpu')        
    # model’s parameters
    model.load_state_dict(state_dict['model'])
    print('Loaded model from epoch {epoch}, step {step:,}'.format(**state_dict))
    model.eval()
In [7]:
test_transform = Compose([
    Resize((SIZE, SIZE)),
    RandomHorizontalFlip(0.5),
  #  CenterCrop(SIZE),
  #  RandomCrop(SIZE),
  #  ColorJitter(brightness=(0.6, 1.4), contrast=0, saturation=0, hue=0),
  #  ColorJitter(brightness=0, contrast=(0.6, 1.4), saturation=0, hue=0),
  #  ColorJitter(brightness=0, contrast=0, saturation=(0.6, 1.4), hue=0),  
  #  ColorJitter(brightness=0, contrast=0, saturation=0, hue=(-0.2, 0.2)), 
  #  RandomGrayscale(p=0.2),    
])    

tensor_transform = Compose([
    ToTensor(),
    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
In [8]:
def load_image(item, root: Path) -> Image.Image:
    image = cv2.imread(str(root / f'{item.id}.png'))
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    return Image.fromarray(image)

def get_ids(root: Path) -> List[str]:
    return sorted({p.name.split('_')[0] for p in root.glob('*.png')})

def mean_df(df: pd.DataFrame) -> pd.DataFrame:
    return df.groupby(level=0).mean()
In [9]:
"""
Datasets
""" 
class TTADataset(Dataset):
    def __init__(self, root: Path, df: pd.DataFrame,
                 image_transform: Callable, tta: int):
        super().__init__()
        self._root = root
        self._df = df
        self._image_transform = image_transform
        self._tta = tta       

    def __len__(self):
        return len(self._df) * self._tta

    def __getitem__(self, idx: int):
        item = self._df.iloc[idx % len(self._df)]
        image = load_image(item, self._root)  
        
        width, height = image.size             
        if height < 320:
            ratio = 400/height
            image = image.resize((int(width * ratio), int(height * ratio)), Image.ANTIALIAS)            
        if width < 320:
            ratio = 400/width
            image = image.resize((int(width * ratio), int(height * ratio)), Image.ANTIALIAS)     
            
        image = self._image_transform(image)
        image = tensor_transform(image)    
        return image, item.id   
In [10]:
def predict(model, root: Path, predict_df: pd.DataFrame, save_root: str,
            image_transform, batch_size: int, tta: int, workers: int, use_cuda: bool):
    """
    Make and save preditions
    """    
    valid_loader = DataLoader(
                TTADataset(root, predict_df, image_transform, tta),
                shuffle=False,
                batch_size=batch_size,
                num_workers=workers,
                )
    print(f'{len(valid_loader.dataset):,} in valid')
    
    all_outputs, all_ids = [], []        
    with torch.no_grad():
        for inputs, ids in tqdm.tqdm(valid_loader, desc='Predict'):
            if use_cuda:
                inputs = inputs.cuda()
            outputs = torch.sigmoid(model(inputs))
            all_outputs.append(outputs.data.cpu().numpy())
            all_ids.extend(ids)    
    # print(all_outputs) 
    df = pd.DataFrame(
            data=np.concatenate(all_outputs),
            index=all_ids,
            columns=map(str, range(N_CLASSES)))
    df = mean_df(df)
    print('probs: ', df.head(10))    
    #df.to_hdf(save_root, 'prob', index_label='id')
    #print(f'Saved predictions to {out_path}')
    return df
       
        
def binarize_prediction(probabilities, threshold: float, argsorted=None,
                        min_labels=1, max_labels=10):
    """ Return matrix of 0/1 predictions, same shape as probabilities.
    """
    assert probabilities.shape[1] == N_CLASSES
    if argsorted is None:
        argsorted = probabilities.argsort(axis=1)
    max_mask = _make_mask(argsorted, max_labels)
    min_mask = _make_mask(argsorted, min_labels)
    prob_mask = probabilities > threshold
    return (max_mask & prob_mask) | min_mask


def _make_mask(argsorted, top_n: int):
    mask = np.zeros_like(argsorted, dtype=np.uint8)
    col_indices = argsorted[:, -top_n:].reshape(-1)
    row_indices = [i // top_n for i in range(len(col_indices))]
    mask[row_indices, col_indices] = 1
    return mask
In [11]:
predict_kwargs = dict(
        image_transform = test_transform,
        batch_size=16,
        tta=2,
        workers=0,
        use_cuda=use_cuda
        )    

ss = pd.read_csv(DATA_ROOT/'sample_submission.csv')
if use_sample:
    ss = ss.head(100)     
print(ss.head())
                 id attribute_ids
0  10023b2cc4ed5f68         0 1 2
1  100fbe75ed8fd887         0 1 2
2  101b627524a04f19         0 1 2
3  10234480c41284c6         0 1 2
4  1023b0e2636dcea8         0 1 2
In [12]:
sample_submission = pd.read_csv(
        DATA_ROOT / 'sample_submission.csv', index_col='id')

def get_classes(item):
    return ' '.join(cls for cls, is_present in item.items() if is_present)

dfs = []
predictions = []
folds = [0, 11, 2, 3, 4]
for fold in folds:
    load_model(model, RUN_ROOT, fold, use_cuda)
    out_path='test_'+str(fold)+'.h5'  
    df = predict(model, test_root, ss, out_path, **predict_kwargs)
    df = df.reindex(sample_submission.index)
    dfs.append(df)
    print(dfs)
        
df = pd.concat(dfs)
print(df.head())
# average 5 folds
df = mean_df(df)
df[:] = binarize_prediction(df.values, threshold=0.11)
df = df.apply(get_classes, axis=1)
df.name = 'attribute_ids'
df.to_csv('submission.csv', header=True)
print(df.head()) 
Predict:   0%|          | 0/931 [00:00<?, ?it/s]
Loaded model from epoch 9, step 43,712
14,886 in valid
Predict: 100%|██████████| 931/931 [07:34<00:00,  2.45it/s]
probs:                               0      ...               1102
10023b2cc4ed5f68  4.728668e-13      ...       1.030075e-10
100fbe75ed8fd887  3.072049e-14      ...       4.280107e-07
101b627524a04f19  2.371050e-10      ...       7.791431e-05
10234480c41284c6  9.136196e-11      ...       2.918332e-08
1023b0e2636dcea8  3.340438e-12      ...       1.591823e-08
1039cd6cf85845c   4.337629e-15      ...       1.039632e-09
103a5b3f83fbe88   1.340217e-08      ...       3.086209e-05
10413aaae8d6a9a2  7.633454e-10      ...       4.874447e-07
10423822b93a65ab  1.558484e-09      ...       5.887860e-07
1052bf702cb099f7  5.377024e-14      ...       1.300073e-04

[10 rows x 1103 columns]
[                             0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.728668e-13      ...       1.030075e-10
100fbe75ed8fd887  3.072049e-14      ...       4.280107e-07
101b627524a04f19  2.371050e-10      ...       7.791431e-05
10234480c41284c6  9.136196e-11      ...       2.918332e-08
1023b0e2636dcea8  3.340438e-12      ...       1.591823e-08
1039cd6cf85845c   4.337629e-15      ...       1.039632e-09
103a5b3f83fbe88   1.340217e-08      ...       3.086209e-05
10413aaae8d6a9a2  7.633454e-10      ...       4.874447e-07
10423822b93a65ab  1.558484e-09      ...       5.887860e-07
1052bf702cb099f7  5.377024e-14      ...       1.300073e-04
10543c918a43a8d   5.157139e-09      ...       7.896976e-09
105c9a3453da79c3  4.366324e-10      ...       3.055582e-07
1060688bbf6eac87  1.208084e-09      ...       5.593986e-07
106a247caeabd15a  1.765468e-10      ...       1.186001e-08
106e21606add59f3  2.761865e-11      ...       9.735194e-11
107c38495881b6c9  1.366474e-06      ...       3.426261e-05
108815dd3752ab64  4.891811e-15      ...       5.944660e-08
10943defdd5d5e89  6.925465e-12      ...       6.808719e-06
10a39a78c44ef27c  8.010963e-07      ...       3.032649e-04
10ab70df067bdb4   4.453399e-11      ...       2.154509e-11
10b28e3de3566582  2.334930e-08      ...       4.971076e-06
10b32964331a6cc3  5.610753e-09      ...       1.369045e-06
10b4562e7fa6f668  6.153600e-08      ...       2.608227e-05
10db1c338e1d822f  6.803167e-10      ...       3.939526e-07
10e0c215f5f3084e  2.529129e-16      ...       4.029422e-16
10e95bead8e0b35b  9.851347e-12      ...       6.908680e-07
1100d7b0f24fee88  1.280776e-08      ...       6.198838e-06
11099b321e8c7066  2.435142e-08      ...       1.358786e-06
110df388fd5c50e4  6.032639e-10      ...       4.897266e-07
113520ea0138f76d  1.231125e-09      ...       1.025111e-04
...                        ...      ...                ...
ff2da1f0ed3e3ebe  2.739189e-26      ...       1.377843e-14
ff3a9fa43f8eab9c  2.253703e-14      ...       1.077716e-06
ff44490e20740a19  9.080540e-10      ...       3.801882e-06
ff481bb029678d5d  5.033446e-10      ...       9.471140e-07
ff4c3570fb7b90d3  2.584622e-07      ...       2.476596e-08
ff4f548d08414709  8.068520e-07      ...       5.496110e-11
ff668377a518ea5f  1.695071e-09      ...       7.228905e-06
ff6a549b2d7a0e76  6.122934e-08      ...       1.515677e-06
ff6ee1b37c8dc1ae  7.569311e-12      ...       1.531516e-07
ff85460d6b853b49  1.644709e-09      ...       6.753521e-10
ff8721b85d1b5a5   1.707178e-15      ...       3.052756e-11
ff8bef7d0de52b31  8.941545e-04      ...       7.484565e-10
ff92504c82c41e0f  1.490968e-11      ...       1.039263e-05
ff9d4b77c124c9f2  8.088179e-13      ...       2.885554e-08
ff9ddf70cb1c2674  1.476271e-08      ...       3.213697e-05
ffaf8c3fe0b1d9b6  2.524084e-16      ...       7.134760e-09
ffb61df4a6734772  2.452295e-18      ...       3.635929e-11
ffb73f95b8721900  3.496576e-17      ...       3.272325e-12
ffb937b55755323e  6.148928e-09      ...       2.119681e-07
ffbcf8b91a8e8ce0  3.936214e-10      ...       4.039476e-07
ffbf4849bde21b0a  1.047900e-10      ...       2.959225e-06
ffc96e053345419d  1.336849e-09      ...       1.727152e-07
ffcb16053099d795  9.935106e-10      ...       1.743506e-07
ffcf745289465074  4.772135e-10      ...       7.184802e-06
ffd1372fe67e65f0  9.845096e-24      ...       1.447992e-19
ffd79eadf642221b  7.616045e-12      ...       2.628909e-10
ffd96986aa333f4d  1.433656e-09      ...       1.730341e-05
ffe54b454396d97c  1.886243e-06      ...       1.414021e-06
ffe7d7db4e4aa37f  1.040358e-02      ...       6.017941e-10
ffed0a4aca0d5457  1.113444e-12      ...       4.861438e-09

[7443 rows x 1103 columns]]
Predict:   0%|          | 0/931 [00:00<?, ?it/s]
Loaded model from epoch 9, step 43,680
14,886 in valid
Predict: 100%|██████████| 931/931 [07:21<00:00,  2.52it/s]
probs:                               0      ...               1102
10023b2cc4ed5f68  4.169688e-12      ...       9.107919e-11
100fbe75ed8fd887  2.094033e-09      ...       9.253769e-07
101b627524a04f19  8.167342e-15      ...       2.280261e-06
10234480c41284c6  4.473180e-10      ...       1.535988e-08
1023b0e2636dcea8  6.650800e-11      ...       2.151765e-07
1039cd6cf85845c   4.996251e-16      ...       1.122741e-09
103a5b3f83fbe88   1.916447e-11      ...       4.996276e-07
10413aaae8d6a9a2  4.700306e-09      ...       4.433651e-07
10423822b93a65ab  2.013114e-07      ...       2.491866e-07
1052bf702cb099f7  2.040713e-10      ...       2.565025e-05

[10 rows x 1103 columns]
[                             0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.728668e-13      ...       1.030075e-10
100fbe75ed8fd887  3.072049e-14      ...       4.280107e-07
101b627524a04f19  2.371050e-10      ...       7.791431e-05
10234480c41284c6  9.136196e-11      ...       2.918332e-08
1023b0e2636dcea8  3.340438e-12      ...       1.591823e-08
1039cd6cf85845c   4.337629e-15      ...       1.039632e-09
103a5b3f83fbe88   1.340217e-08      ...       3.086209e-05
10413aaae8d6a9a2  7.633454e-10      ...       4.874447e-07
10423822b93a65ab  1.558484e-09      ...       5.887860e-07
1052bf702cb099f7  5.377024e-14      ...       1.300073e-04
10543c918a43a8d   5.157139e-09      ...       7.896976e-09
105c9a3453da79c3  4.366324e-10      ...       3.055582e-07
1060688bbf6eac87  1.208084e-09      ...       5.593986e-07
106a247caeabd15a  1.765468e-10      ...       1.186001e-08
106e21606add59f3  2.761865e-11      ...       9.735194e-11
107c38495881b6c9  1.366474e-06      ...       3.426261e-05
108815dd3752ab64  4.891811e-15      ...       5.944660e-08
10943defdd5d5e89  6.925465e-12      ...       6.808719e-06
10a39a78c44ef27c  8.010963e-07      ...       3.032649e-04
10ab70df067bdb4   4.453399e-11      ...       2.154509e-11
10b28e3de3566582  2.334930e-08      ...       4.971076e-06
10b32964331a6cc3  5.610753e-09      ...       1.369045e-06
10b4562e7fa6f668  6.153600e-08      ...       2.608227e-05
10db1c338e1d822f  6.803167e-10      ...       3.939526e-07
10e0c215f5f3084e  2.529129e-16      ...       4.029422e-16
10e95bead8e0b35b  9.851347e-12      ...       6.908680e-07
1100d7b0f24fee88  1.280776e-08      ...       6.198838e-06
11099b321e8c7066  2.435142e-08      ...       1.358786e-06
110df388fd5c50e4  6.032639e-10      ...       4.897266e-07
113520ea0138f76d  1.231125e-09      ...       1.025111e-04
...                        ...      ...                ...
ff2da1f0ed3e3ebe  2.739189e-26      ...       1.377843e-14
ff3a9fa43f8eab9c  2.253703e-14      ...       1.077716e-06
ff44490e20740a19  9.080540e-10      ...       3.801882e-06
ff481bb029678d5d  5.033446e-10      ...       9.471140e-07
ff4c3570fb7b90d3  2.584622e-07      ...       2.476596e-08
ff4f548d08414709  8.068520e-07      ...       5.496110e-11
ff668377a518ea5f  1.695071e-09      ...       7.228905e-06
ff6a549b2d7a0e76  6.122934e-08      ...       1.515677e-06
ff6ee1b37c8dc1ae  7.569311e-12      ...       1.531516e-07
ff85460d6b853b49  1.644709e-09      ...       6.753521e-10
ff8721b85d1b5a5   1.707178e-15      ...       3.052756e-11
ff8bef7d0de52b31  8.941545e-04      ...       7.484565e-10
ff92504c82c41e0f  1.490968e-11      ...       1.039263e-05
ff9d4b77c124c9f2  8.088179e-13      ...       2.885554e-08
ff9ddf70cb1c2674  1.476271e-08      ...       3.213697e-05
ffaf8c3fe0b1d9b6  2.524084e-16      ...       7.134760e-09
ffb61df4a6734772  2.452295e-18      ...       3.635929e-11
ffb73f95b8721900  3.496576e-17      ...       3.272325e-12
ffb937b55755323e  6.148928e-09      ...       2.119681e-07
ffbcf8b91a8e8ce0  3.936214e-10      ...       4.039476e-07
ffbf4849bde21b0a  1.047900e-10      ...       2.959225e-06
ffc96e053345419d  1.336849e-09      ...       1.727152e-07
ffcb16053099d795  9.935106e-10      ...       1.743506e-07
ffcf745289465074  4.772135e-10      ...       7.184802e-06
ffd1372fe67e65f0  9.845096e-24      ...       1.447992e-19
ffd79eadf642221b  7.616045e-12      ...       2.628909e-10
ffd96986aa333f4d  1.433656e-09      ...       1.730341e-05
ffe54b454396d97c  1.886243e-06      ...       1.414021e-06
ffe7d7db4e4aa37f  1.040358e-02      ...       6.017941e-10
ffed0a4aca0d5457  1.113444e-12      ...       4.861438e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.169688e-12      ...       9.107919e-11
100fbe75ed8fd887  2.094033e-09      ...       9.253769e-07
101b627524a04f19  8.167342e-15      ...       2.280261e-06
10234480c41284c6  4.473180e-10      ...       1.535988e-08
1023b0e2636dcea8  6.650800e-11      ...       2.151765e-07
1039cd6cf85845c   4.996251e-16      ...       1.122741e-09
103a5b3f83fbe88   1.916447e-11      ...       4.996276e-07
10413aaae8d6a9a2  4.700306e-09      ...       4.433651e-07
10423822b93a65ab  2.013114e-07      ...       2.491866e-07
1052bf702cb099f7  2.040713e-10      ...       2.565025e-05
10543c918a43a8d   3.331643e-08      ...       8.023857e-07
105c9a3453da79c3  8.417312e-13      ...       1.297822e-06
1060688bbf6eac87  3.050659e-09      ...       7.936198e-06
106a247caeabd15a  1.402277e-13      ...       1.042819e-11
106e21606add59f3  2.030312e-10      ...       1.132089e-09
107c38495881b6c9  2.989174e-08      ...       2.510220e-05
108815dd3752ab64  1.272666e-19      ...       5.041187e-11
10943defdd5d5e89  6.676304e-09      ...       2.253270e-05
10a39a78c44ef27c  1.378666e-07      ...       2.025353e-04
10ab70df067bdb4   1.070075e-11      ...       1.749080e-12
10b28e3de3566582  9.555376e-12      ...       7.372234e-09
10b32964331a6cc3  1.762800e-09      ...       2.154288e-06
10b4562e7fa6f668  4.061142e-08      ...       6.333739e-06
10db1c338e1d822f  6.489576e-09      ...       6.065164e-06
10e0c215f5f3084e  2.604376e-14      ...       3.198323e-10
10e95bead8e0b35b  1.891171e-14      ...       7.382744e-09
1100d7b0f24fee88  3.096013e-08      ...       3.619512e-06
11099b321e8c7066  1.361922e-08      ...       3.388539e-05
110df388fd5c50e4  2.705576e-09      ...       1.869140e-06
113520ea0138f76d  5.520975e-09      ...       1.418945e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  4.393324e-17      ...       2.235492e-10
ff3a9fa43f8eab9c  3.861345e-12      ...       6.479478e-06
ff44490e20740a19  7.186737e-10      ...       1.935095e-07
ff481bb029678d5d  1.325141e-09      ...       1.202223e-07
ff4c3570fb7b90d3  2.790299e-08      ...       1.199169e-09
ff4f548d08414709  8.061184e-07      ...       2.487089e-09
ff668377a518ea5f  9.763571e-09      ...       1.240066e-06
ff6a549b2d7a0e76  7.110477e-07      ...       2.132758e-05
ff6ee1b37c8dc1ae  8.808906e-09      ...       2.684915e-06
ff85460d6b853b49  2.092690e-07      ...       3.347819e-07
ff8721b85d1b5a5   1.739395e-13      ...       1.368489e-10
ff8bef7d0de52b31  7.612832e-05      ...       3.112961e-11
ff92504c82c41e0f  6.241245e-13      ...       3.171968e-07
ff9d4b77c124c9f2  2.353988e-09      ...       5.624191e-08
ff9ddf70cb1c2674  7.063157e-10      ...       3.009483e-06
ffaf8c3fe0b1d9b6  2.541497e-12      ...       6.226065e-10
ffb61df4a6734772  1.097592e-26      ...       3.128824e-19
ffb73f95b8721900  3.289229e-18      ...       1.881861e-12
ffb937b55755323e  3.094228e-07      ...       2.350395e-05
ffbcf8b91a8e8ce0  2.189928e-12      ...       7.289162e-09
ffbf4849bde21b0a  4.982286e-13      ...       4.595518e-07
ffc96e053345419d  8.103861e-11      ...       3.864488e-08
ffcb16053099d795  2.100295e-09      ...       2.070636e-07
ffcf745289465074  1.453044e-10      ...       2.270694e-06
ffd1372fe67e65f0  8.377085e-22      ...       2.415360e-21
ffd79eadf642221b  2.746374e-13      ...       3.338333e-10
ffd96986aa333f4d  7.501828e-10      ...       6.431012e-06
ffe54b454396d97c  4.627787e-07      ...       3.034937e-07
ffe7d7db4e4aa37f  3.006166e-02      ...       3.039038e-08
ffed0a4aca0d5457  6.005101e-13      ...       6.678660e-09

[7443 rows x 1103 columns]]
Predict:   0%|          | 0/931 [00:00<?, ?it/s]
Loaded model from epoch 9, step 43,680
14,886 in valid
Predict: 100%|██████████| 931/931 [07:20<00:00,  2.48it/s]
probs:                               0      ...               1102
10023b2cc4ed5f68  1.688872e-12      ...       1.233552e-08
100fbe75ed8fd887  8.501226e-18      ...       2.928678e-13
101b627524a04f19  2.754502e-16      ...       1.895944e-06
10234480c41284c6  6.239672e-10      ...       2.116354e-09
1023b0e2636dcea8  5.640889e-11      ...       3.327427e-09
1039cd6cf85845c   5.316236e-14      ...       1.757189e-11
103a5b3f83fbe88   3.230017e-11      ...       5.635370e-08
10413aaae8d6a9a2  1.347011e-09      ...       2.597413e-08
10423822b93a65ab  2.850066e-08      ...       1.002683e-06
1052bf702cb099f7  1.327797e-14      ...       1.762457e-04

[10 rows x 1103 columns]
[                             0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.728668e-13      ...       1.030075e-10
100fbe75ed8fd887  3.072049e-14      ...       4.280107e-07
101b627524a04f19  2.371050e-10      ...       7.791431e-05
10234480c41284c6  9.136196e-11      ...       2.918332e-08
1023b0e2636dcea8  3.340438e-12      ...       1.591823e-08
1039cd6cf85845c   4.337629e-15      ...       1.039632e-09
103a5b3f83fbe88   1.340217e-08      ...       3.086209e-05
10413aaae8d6a9a2  7.633454e-10      ...       4.874447e-07
10423822b93a65ab  1.558484e-09      ...       5.887860e-07
1052bf702cb099f7  5.377024e-14      ...       1.300073e-04
10543c918a43a8d   5.157139e-09      ...       7.896976e-09
105c9a3453da79c3  4.366324e-10      ...       3.055582e-07
1060688bbf6eac87  1.208084e-09      ...       5.593986e-07
106a247caeabd15a  1.765468e-10      ...       1.186001e-08
106e21606add59f3  2.761865e-11      ...       9.735194e-11
107c38495881b6c9  1.366474e-06      ...       3.426261e-05
108815dd3752ab64  4.891811e-15      ...       5.944660e-08
10943defdd5d5e89  6.925465e-12      ...       6.808719e-06
10a39a78c44ef27c  8.010963e-07      ...       3.032649e-04
10ab70df067bdb4   4.453399e-11      ...       2.154509e-11
10b28e3de3566582  2.334930e-08      ...       4.971076e-06
10b32964331a6cc3  5.610753e-09      ...       1.369045e-06
10b4562e7fa6f668  6.153600e-08      ...       2.608227e-05
10db1c338e1d822f  6.803167e-10      ...       3.939526e-07
10e0c215f5f3084e  2.529129e-16      ...       4.029422e-16
10e95bead8e0b35b  9.851347e-12      ...       6.908680e-07
1100d7b0f24fee88  1.280776e-08      ...       6.198838e-06
11099b321e8c7066  2.435142e-08      ...       1.358786e-06
110df388fd5c50e4  6.032639e-10      ...       4.897266e-07
113520ea0138f76d  1.231125e-09      ...       1.025111e-04
...                        ...      ...                ...
ff2da1f0ed3e3ebe  2.739189e-26      ...       1.377843e-14
ff3a9fa43f8eab9c  2.253703e-14      ...       1.077716e-06
ff44490e20740a19  9.080540e-10      ...       3.801882e-06
ff481bb029678d5d  5.033446e-10      ...       9.471140e-07
ff4c3570fb7b90d3  2.584622e-07      ...       2.476596e-08
ff4f548d08414709  8.068520e-07      ...       5.496110e-11
ff668377a518ea5f  1.695071e-09      ...       7.228905e-06
ff6a549b2d7a0e76  6.122934e-08      ...       1.515677e-06
ff6ee1b37c8dc1ae  7.569311e-12      ...       1.531516e-07
ff85460d6b853b49  1.644709e-09      ...       6.753521e-10
ff8721b85d1b5a5   1.707178e-15      ...       3.052756e-11
ff8bef7d0de52b31  8.941545e-04      ...       7.484565e-10
ff92504c82c41e0f  1.490968e-11      ...       1.039263e-05
ff9d4b77c124c9f2  8.088179e-13      ...       2.885554e-08
ff9ddf70cb1c2674  1.476271e-08      ...       3.213697e-05
ffaf8c3fe0b1d9b6  2.524084e-16      ...       7.134760e-09
ffb61df4a6734772  2.452295e-18      ...       3.635929e-11
ffb73f95b8721900  3.496576e-17      ...       3.272325e-12
ffb937b55755323e  6.148928e-09      ...       2.119681e-07
ffbcf8b91a8e8ce0  3.936214e-10      ...       4.039476e-07
ffbf4849bde21b0a  1.047900e-10      ...       2.959225e-06
ffc96e053345419d  1.336849e-09      ...       1.727152e-07
ffcb16053099d795  9.935106e-10      ...       1.743506e-07
ffcf745289465074  4.772135e-10      ...       7.184802e-06
ffd1372fe67e65f0  9.845096e-24      ...       1.447992e-19
ffd79eadf642221b  7.616045e-12      ...       2.628909e-10
ffd96986aa333f4d  1.433656e-09      ...       1.730341e-05
ffe54b454396d97c  1.886243e-06      ...       1.414021e-06
ffe7d7db4e4aa37f  1.040358e-02      ...       6.017941e-10
ffed0a4aca0d5457  1.113444e-12      ...       4.861438e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.169688e-12      ...       9.107919e-11
100fbe75ed8fd887  2.094033e-09      ...       9.253769e-07
101b627524a04f19  8.167342e-15      ...       2.280261e-06
10234480c41284c6  4.473180e-10      ...       1.535988e-08
1023b0e2636dcea8  6.650800e-11      ...       2.151765e-07
1039cd6cf85845c   4.996251e-16      ...       1.122741e-09
103a5b3f83fbe88   1.916447e-11      ...       4.996276e-07
10413aaae8d6a9a2  4.700306e-09      ...       4.433651e-07
10423822b93a65ab  2.013114e-07      ...       2.491866e-07
1052bf702cb099f7  2.040713e-10      ...       2.565025e-05
10543c918a43a8d   3.331643e-08      ...       8.023857e-07
105c9a3453da79c3  8.417312e-13      ...       1.297822e-06
1060688bbf6eac87  3.050659e-09      ...       7.936198e-06
106a247caeabd15a  1.402277e-13      ...       1.042819e-11
106e21606add59f3  2.030312e-10      ...       1.132089e-09
107c38495881b6c9  2.989174e-08      ...       2.510220e-05
108815dd3752ab64  1.272666e-19      ...       5.041187e-11
10943defdd5d5e89  6.676304e-09      ...       2.253270e-05
10a39a78c44ef27c  1.378666e-07      ...       2.025353e-04
10ab70df067bdb4   1.070075e-11      ...       1.749080e-12
10b28e3de3566582  9.555376e-12      ...       7.372234e-09
10b32964331a6cc3  1.762800e-09      ...       2.154288e-06
10b4562e7fa6f668  4.061142e-08      ...       6.333739e-06
10db1c338e1d822f  6.489576e-09      ...       6.065164e-06
10e0c215f5f3084e  2.604376e-14      ...       3.198323e-10
10e95bead8e0b35b  1.891171e-14      ...       7.382744e-09
1100d7b0f24fee88  3.096013e-08      ...       3.619512e-06
11099b321e8c7066  1.361922e-08      ...       3.388539e-05
110df388fd5c50e4  2.705576e-09      ...       1.869140e-06
113520ea0138f76d  5.520975e-09      ...       1.418945e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  4.393324e-17      ...       2.235492e-10
ff3a9fa43f8eab9c  3.861345e-12      ...       6.479478e-06
ff44490e20740a19  7.186737e-10      ...       1.935095e-07
ff481bb029678d5d  1.325141e-09      ...       1.202223e-07
ff4c3570fb7b90d3  2.790299e-08      ...       1.199169e-09
ff4f548d08414709  8.061184e-07      ...       2.487089e-09
ff668377a518ea5f  9.763571e-09      ...       1.240066e-06
ff6a549b2d7a0e76  7.110477e-07      ...       2.132758e-05
ff6ee1b37c8dc1ae  8.808906e-09      ...       2.684915e-06
ff85460d6b853b49  2.092690e-07      ...       3.347819e-07
ff8721b85d1b5a5   1.739395e-13      ...       1.368489e-10
ff8bef7d0de52b31  7.612832e-05      ...       3.112961e-11
ff92504c82c41e0f  6.241245e-13      ...       3.171968e-07
ff9d4b77c124c9f2  2.353988e-09      ...       5.624191e-08
ff9ddf70cb1c2674  7.063157e-10      ...       3.009483e-06
ffaf8c3fe0b1d9b6  2.541497e-12      ...       6.226065e-10
ffb61df4a6734772  1.097592e-26      ...       3.128824e-19
ffb73f95b8721900  3.289229e-18      ...       1.881861e-12
ffb937b55755323e  3.094228e-07      ...       2.350395e-05
ffbcf8b91a8e8ce0  2.189928e-12      ...       7.289162e-09
ffbf4849bde21b0a  4.982286e-13      ...       4.595518e-07
ffc96e053345419d  8.103861e-11      ...       3.864488e-08
ffcb16053099d795  2.100295e-09      ...       2.070636e-07
ffcf745289465074  1.453044e-10      ...       2.270694e-06
ffd1372fe67e65f0  8.377085e-22      ...       2.415360e-21
ffd79eadf642221b  2.746374e-13      ...       3.338333e-10
ffd96986aa333f4d  7.501828e-10      ...       6.431012e-06
ffe54b454396d97c  4.627787e-07      ...       3.034937e-07
ffe7d7db4e4aa37f  3.006166e-02      ...       3.039038e-08
ffed0a4aca0d5457  6.005101e-13      ...       6.678660e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  1.688872e-12      ...       1.233552e-08
100fbe75ed8fd887  8.501226e-18      ...       2.928678e-13
101b627524a04f19  2.754502e-16      ...       1.895944e-06
10234480c41284c6  6.239672e-10      ...       2.116354e-09
1023b0e2636dcea8  5.640889e-11      ...       3.327427e-09
1039cd6cf85845c   5.316236e-14      ...       1.757189e-11
103a5b3f83fbe88   3.230017e-11      ...       5.635370e-08
10413aaae8d6a9a2  1.347011e-09      ...       2.597413e-08
10423822b93a65ab  2.850066e-08      ...       1.002683e-06
1052bf702cb099f7  1.327797e-14      ...       1.762457e-04
10543c918a43a8d   1.374395e-08      ...       4.425313e-06
105c9a3453da79c3  1.803370e-10      ...       4.606464e-06
1060688bbf6eac87  1.407711e-08      ...       1.245683e-05
106a247caeabd15a  2.377964e-09      ...       1.076588e-06
106e21606add59f3  1.502047e-10      ...       5.570284e-10
107c38495881b6c9  3.697720e-07      ...       2.185369e-06
108815dd3752ab64  2.567115e-16      ...       6.964844e-09
10943defdd5d5e89  3.195673e-13      ...       6.107219e-06
10a39a78c44ef27c  3.378688e-07      ...       8.190519e-05
10ab70df067bdb4   9.783815e-09      ...       1.388995e-09
10b28e3de3566582  8.516890e-12      ...       1.151808e-05
10b32964331a6cc3  5.361789e-09      ...       1.095121e-05
10b4562e7fa6f668  2.074001e-10      ...       1.835770e-05
10db1c338e1d822f  2.169353e-08      ...       4.518313e-06
10e0c215f5f3084e  2.484049e-09      ...       2.299547e-08
10e95bead8e0b35b  7.013370e-12      ...       1.277378e-06
1100d7b0f24fee88  2.016052e-08      ...       1.259897e-05
11099b321e8c7066  1.185949e-07      ...       2.246400e-05
110df388fd5c50e4  5.730570e-09      ...       2.721508e-06
113520ea0138f76d  6.833046e-09      ...       2.664440e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  1.217164e-17      ...       2.189270e-11
ff3a9fa43f8eab9c  7.947099e-16      ...       1.604736e-05
ff44490e20740a19  4.647956e-09      ...       6.015821e-05
ff481bb029678d5d  9.050901e-10      ...       9.972478e-08
ff4c3570fb7b90d3  4.873484e-05      ...       1.685198e-07
ff4f548d08414709  2.159410e-08      ...       4.998338e-13
ff668377a518ea5f  4.068058e-10      ...       1.245732e-06
ff6a549b2d7a0e76  3.591847e-06      ...       6.030097e-07
ff6ee1b37c8dc1ae  2.036120e-08      ...       3.787195e-05
ff85460d6b853b49  3.245934e-09      ...       5.576711e-12
ff8721b85d1b5a5   4.492046e-19      ...       6.564167e-16
ff8bef7d0de52b31  6.228408e-03      ...       2.351988e-09
ff92504c82c41e0f  4.867560e-13      ...       1.299587e-07
ff9d4b77c124c9f2  2.684753e-10      ...       4.635839e-05
ff9ddf70cb1c2674  4.570826e-10      ...       7.221037e-09
ffaf8c3fe0b1d9b6  1.269481e-15      ...       5.569583e-10
ffb61df4a6734772  1.189174e-17      ...       6.532761e-14
ffb73f95b8721900  1.816950e-15      ...       9.251801e-12
ffb937b55755323e  2.318472e-07      ...       1.706806e-05
ffbcf8b91a8e8ce0  9.387164e-12      ...       3.274495e-10
ffbf4849bde21b0a  1.933418e-13      ...       8.629256e-09
ffc96e053345419d  4.242912e-11      ...       1.367808e-07
ffcb16053099d795  7.353135e-11      ...       3.171480e-10
ffcf745289465074  1.550135e-09      ...       6.976936e-05
ffd1372fe67e65f0  7.258660e-22      ...       9.557752e-24
ffd79eadf642221b  8.458958e-13      ...       3.482113e-11
ffd96986aa333f4d  1.167264e-09      ...       2.022440e-05
ffe54b454396d97c  1.774958e-04      ...       1.121146e-06
ffe7d7db4e4aa37f  1.224639e-02      ...       3.347652e-08
ffed0a4aca0d5457  8.636182e-14      ...       2.780588e-09

[7443 rows x 1103 columns]]
Predict:   0%|          | 0/931 [00:00<?, ?it/s]
Loaded model from epoch 7, step 32,766
14,886 in valid
Predict: 100%|██████████| 931/931 [07:22<00:00,  2.50it/s]
probs:                               0      ...               1102
10023b2cc4ed5f68  4.252782e-09      ...       8.373580e-09
100fbe75ed8fd887  1.338250e-13      ...       3.233474e-11
101b627524a04f19  1.567435e-12      ...       1.852006e-05
10234480c41284c6  1.993155e-08      ...       1.661433e-06
1023b0e2636dcea8  4.199389e-10      ...       6.184982e-07
1039cd6cf85845c   1.083505e-10      ...       7.835828e-08
103a5b3f83fbe88   1.772693e-10      ...       2.923270e-06
10413aaae8d6a9a2  8.621997e-09      ...       1.186972e-05
10423822b93a65ab  3.379614e-08      ...       5.833151e-08
1052bf702cb099f7  3.944712e-12      ...       4.587294e-07

[10 rows x 1103 columns]
[                             0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.728668e-13      ...       1.030075e-10
100fbe75ed8fd887  3.072049e-14      ...       4.280107e-07
101b627524a04f19  2.371050e-10      ...       7.791431e-05
10234480c41284c6  9.136196e-11      ...       2.918332e-08
1023b0e2636dcea8  3.340438e-12      ...       1.591823e-08
1039cd6cf85845c   4.337629e-15      ...       1.039632e-09
103a5b3f83fbe88   1.340217e-08      ...       3.086209e-05
10413aaae8d6a9a2  7.633454e-10      ...       4.874447e-07
10423822b93a65ab  1.558484e-09      ...       5.887860e-07
1052bf702cb099f7  5.377024e-14      ...       1.300073e-04
10543c918a43a8d   5.157139e-09      ...       7.896976e-09
105c9a3453da79c3  4.366324e-10      ...       3.055582e-07
1060688bbf6eac87  1.208084e-09      ...       5.593986e-07
106a247caeabd15a  1.765468e-10      ...       1.186001e-08
106e21606add59f3  2.761865e-11      ...       9.735194e-11
107c38495881b6c9  1.366474e-06      ...       3.426261e-05
108815dd3752ab64  4.891811e-15      ...       5.944660e-08
10943defdd5d5e89  6.925465e-12      ...       6.808719e-06
10a39a78c44ef27c  8.010963e-07      ...       3.032649e-04
10ab70df067bdb4   4.453399e-11      ...       2.154509e-11
10b28e3de3566582  2.334930e-08      ...       4.971076e-06
10b32964331a6cc3  5.610753e-09      ...       1.369045e-06
10b4562e7fa6f668  6.153600e-08      ...       2.608227e-05
10db1c338e1d822f  6.803167e-10      ...       3.939526e-07
10e0c215f5f3084e  2.529129e-16      ...       4.029422e-16
10e95bead8e0b35b  9.851347e-12      ...       6.908680e-07
1100d7b0f24fee88  1.280776e-08      ...       6.198838e-06
11099b321e8c7066  2.435142e-08      ...       1.358786e-06
110df388fd5c50e4  6.032639e-10      ...       4.897266e-07
113520ea0138f76d  1.231125e-09      ...       1.025111e-04
...                        ...      ...                ...
ff2da1f0ed3e3ebe  2.739189e-26      ...       1.377843e-14
ff3a9fa43f8eab9c  2.253703e-14      ...       1.077716e-06
ff44490e20740a19  9.080540e-10      ...       3.801882e-06
ff481bb029678d5d  5.033446e-10      ...       9.471140e-07
ff4c3570fb7b90d3  2.584622e-07      ...       2.476596e-08
ff4f548d08414709  8.068520e-07      ...       5.496110e-11
ff668377a518ea5f  1.695071e-09      ...       7.228905e-06
ff6a549b2d7a0e76  6.122934e-08      ...       1.515677e-06
ff6ee1b37c8dc1ae  7.569311e-12      ...       1.531516e-07
ff85460d6b853b49  1.644709e-09      ...       6.753521e-10
ff8721b85d1b5a5   1.707178e-15      ...       3.052756e-11
ff8bef7d0de52b31  8.941545e-04      ...       7.484565e-10
ff92504c82c41e0f  1.490968e-11      ...       1.039263e-05
ff9d4b77c124c9f2  8.088179e-13      ...       2.885554e-08
ff9ddf70cb1c2674  1.476271e-08      ...       3.213697e-05
ffaf8c3fe0b1d9b6  2.524084e-16      ...       7.134760e-09
ffb61df4a6734772  2.452295e-18      ...       3.635929e-11
ffb73f95b8721900  3.496576e-17      ...       3.272325e-12
ffb937b55755323e  6.148928e-09      ...       2.119681e-07
ffbcf8b91a8e8ce0  3.936214e-10      ...       4.039476e-07
ffbf4849bde21b0a  1.047900e-10      ...       2.959225e-06
ffc96e053345419d  1.336849e-09      ...       1.727152e-07
ffcb16053099d795  9.935106e-10      ...       1.743506e-07
ffcf745289465074  4.772135e-10      ...       7.184802e-06
ffd1372fe67e65f0  9.845096e-24      ...       1.447992e-19
ffd79eadf642221b  7.616045e-12      ...       2.628909e-10
ffd96986aa333f4d  1.433656e-09      ...       1.730341e-05
ffe54b454396d97c  1.886243e-06      ...       1.414021e-06
ffe7d7db4e4aa37f  1.040358e-02      ...       6.017941e-10
ffed0a4aca0d5457  1.113444e-12      ...       4.861438e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.169688e-12      ...       9.107919e-11
100fbe75ed8fd887  2.094033e-09      ...       9.253769e-07
101b627524a04f19  8.167342e-15      ...       2.280261e-06
10234480c41284c6  4.473180e-10      ...       1.535988e-08
1023b0e2636dcea8  6.650800e-11      ...       2.151765e-07
1039cd6cf85845c   4.996251e-16      ...       1.122741e-09
103a5b3f83fbe88   1.916447e-11      ...       4.996276e-07
10413aaae8d6a9a2  4.700306e-09      ...       4.433651e-07
10423822b93a65ab  2.013114e-07      ...       2.491866e-07
1052bf702cb099f7  2.040713e-10      ...       2.565025e-05
10543c918a43a8d   3.331643e-08      ...       8.023857e-07
105c9a3453da79c3  8.417312e-13      ...       1.297822e-06
1060688bbf6eac87  3.050659e-09      ...       7.936198e-06
106a247caeabd15a  1.402277e-13      ...       1.042819e-11
106e21606add59f3  2.030312e-10      ...       1.132089e-09
107c38495881b6c9  2.989174e-08      ...       2.510220e-05
108815dd3752ab64  1.272666e-19      ...       5.041187e-11
10943defdd5d5e89  6.676304e-09      ...       2.253270e-05
10a39a78c44ef27c  1.378666e-07      ...       2.025353e-04
10ab70df067bdb4   1.070075e-11      ...       1.749080e-12
10b28e3de3566582  9.555376e-12      ...       7.372234e-09
10b32964331a6cc3  1.762800e-09      ...       2.154288e-06
10b4562e7fa6f668  4.061142e-08      ...       6.333739e-06
10db1c338e1d822f  6.489576e-09      ...       6.065164e-06
10e0c215f5f3084e  2.604376e-14      ...       3.198323e-10
10e95bead8e0b35b  1.891171e-14      ...       7.382744e-09
1100d7b0f24fee88  3.096013e-08      ...       3.619512e-06
11099b321e8c7066  1.361922e-08      ...       3.388539e-05
110df388fd5c50e4  2.705576e-09      ...       1.869140e-06
113520ea0138f76d  5.520975e-09      ...       1.418945e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  4.393324e-17      ...       2.235492e-10
ff3a9fa43f8eab9c  3.861345e-12      ...       6.479478e-06
ff44490e20740a19  7.186737e-10      ...       1.935095e-07
ff481bb029678d5d  1.325141e-09      ...       1.202223e-07
ff4c3570fb7b90d3  2.790299e-08      ...       1.199169e-09
ff4f548d08414709  8.061184e-07      ...       2.487089e-09
ff668377a518ea5f  9.763571e-09      ...       1.240066e-06
ff6a549b2d7a0e76  7.110477e-07      ...       2.132758e-05
ff6ee1b37c8dc1ae  8.808906e-09      ...       2.684915e-06
ff85460d6b853b49  2.092690e-07      ...       3.347819e-07
ff8721b85d1b5a5   1.739395e-13      ...       1.368489e-10
ff8bef7d0de52b31  7.612832e-05      ...       3.112961e-11
ff92504c82c41e0f  6.241245e-13      ...       3.171968e-07
ff9d4b77c124c9f2  2.353988e-09      ...       5.624191e-08
ff9ddf70cb1c2674  7.063157e-10      ...       3.009483e-06
ffaf8c3fe0b1d9b6  2.541497e-12      ...       6.226065e-10
ffb61df4a6734772  1.097592e-26      ...       3.128824e-19
ffb73f95b8721900  3.289229e-18      ...       1.881861e-12
ffb937b55755323e  3.094228e-07      ...       2.350395e-05
ffbcf8b91a8e8ce0  2.189928e-12      ...       7.289162e-09
ffbf4849bde21b0a  4.982286e-13      ...       4.595518e-07
ffc96e053345419d  8.103861e-11      ...       3.864488e-08
ffcb16053099d795  2.100295e-09      ...       2.070636e-07
ffcf745289465074  1.453044e-10      ...       2.270694e-06
ffd1372fe67e65f0  8.377085e-22      ...       2.415360e-21
ffd79eadf642221b  2.746374e-13      ...       3.338333e-10
ffd96986aa333f4d  7.501828e-10      ...       6.431012e-06
ffe54b454396d97c  4.627787e-07      ...       3.034937e-07
ffe7d7db4e4aa37f  3.006166e-02      ...       3.039038e-08
ffed0a4aca0d5457  6.005101e-13      ...       6.678660e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  1.688872e-12      ...       1.233552e-08
100fbe75ed8fd887  8.501226e-18      ...       2.928678e-13
101b627524a04f19  2.754502e-16      ...       1.895944e-06
10234480c41284c6  6.239672e-10      ...       2.116354e-09
1023b0e2636dcea8  5.640889e-11      ...       3.327427e-09
1039cd6cf85845c   5.316236e-14      ...       1.757189e-11
103a5b3f83fbe88   3.230017e-11      ...       5.635370e-08
10413aaae8d6a9a2  1.347011e-09      ...       2.597413e-08
10423822b93a65ab  2.850066e-08      ...       1.002683e-06
1052bf702cb099f7  1.327797e-14      ...       1.762457e-04
10543c918a43a8d   1.374395e-08      ...       4.425313e-06
105c9a3453da79c3  1.803370e-10      ...       4.606464e-06
1060688bbf6eac87  1.407711e-08      ...       1.245683e-05
106a247caeabd15a  2.377964e-09      ...       1.076588e-06
106e21606add59f3  1.502047e-10      ...       5.570284e-10
107c38495881b6c9  3.697720e-07      ...       2.185369e-06
108815dd3752ab64  2.567115e-16      ...       6.964844e-09
10943defdd5d5e89  3.195673e-13      ...       6.107219e-06
10a39a78c44ef27c  3.378688e-07      ...       8.190519e-05
10ab70df067bdb4   9.783815e-09      ...       1.388995e-09
10b28e3de3566582  8.516890e-12      ...       1.151808e-05
10b32964331a6cc3  5.361789e-09      ...       1.095121e-05
10b4562e7fa6f668  2.074001e-10      ...       1.835770e-05
10db1c338e1d822f  2.169353e-08      ...       4.518313e-06
10e0c215f5f3084e  2.484049e-09      ...       2.299547e-08
10e95bead8e0b35b  7.013370e-12      ...       1.277378e-06
1100d7b0f24fee88  2.016052e-08      ...       1.259897e-05
11099b321e8c7066  1.185949e-07      ...       2.246400e-05
110df388fd5c50e4  5.730570e-09      ...       2.721508e-06
113520ea0138f76d  6.833046e-09      ...       2.664440e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  1.217164e-17      ...       2.189270e-11
ff3a9fa43f8eab9c  7.947099e-16      ...       1.604736e-05
ff44490e20740a19  4.647956e-09      ...       6.015821e-05
ff481bb029678d5d  9.050901e-10      ...       9.972478e-08
ff4c3570fb7b90d3  4.873484e-05      ...       1.685198e-07
ff4f548d08414709  2.159410e-08      ...       4.998338e-13
ff668377a518ea5f  4.068058e-10      ...       1.245732e-06
ff6a549b2d7a0e76  3.591847e-06      ...       6.030097e-07
ff6ee1b37c8dc1ae  2.036120e-08      ...       3.787195e-05
ff85460d6b853b49  3.245934e-09      ...       5.576711e-12
ff8721b85d1b5a5   4.492046e-19      ...       6.564167e-16
ff8bef7d0de52b31  6.228408e-03      ...       2.351988e-09
ff92504c82c41e0f  4.867560e-13      ...       1.299587e-07
ff9d4b77c124c9f2  2.684753e-10      ...       4.635839e-05
ff9ddf70cb1c2674  4.570826e-10      ...       7.221037e-09
ffaf8c3fe0b1d9b6  1.269481e-15      ...       5.569583e-10
ffb61df4a6734772  1.189174e-17      ...       6.532761e-14
ffb73f95b8721900  1.816950e-15      ...       9.251801e-12
ffb937b55755323e  2.318472e-07      ...       1.706806e-05
ffbcf8b91a8e8ce0  9.387164e-12      ...       3.274495e-10
ffbf4849bde21b0a  1.933418e-13      ...       8.629256e-09
ffc96e053345419d  4.242912e-11      ...       1.367808e-07
ffcb16053099d795  7.353135e-11      ...       3.171480e-10
ffcf745289465074  1.550135e-09      ...       6.976936e-05
ffd1372fe67e65f0  7.258660e-22      ...       9.557752e-24
ffd79eadf642221b  8.458958e-13      ...       3.482113e-11
ffd96986aa333f4d  1.167264e-09      ...       2.022440e-05
ffe54b454396d97c  1.774958e-04      ...       1.121146e-06
ffe7d7db4e4aa37f  1.224639e-02      ...       3.347652e-08
ffed0a4aca0d5457  8.636182e-14      ...       2.780588e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.252782e-09      ...       8.373580e-09
100fbe75ed8fd887  1.338250e-13      ...       3.233474e-11
101b627524a04f19  1.567435e-12      ...       1.852006e-05
10234480c41284c6  1.993155e-08      ...       1.661433e-06
1023b0e2636dcea8  4.199389e-10      ...       6.184982e-07
1039cd6cf85845c   1.083505e-10      ...       7.835828e-08
103a5b3f83fbe88   1.772693e-10      ...       2.923270e-06
10413aaae8d6a9a2  8.621997e-09      ...       1.186972e-05
10423822b93a65ab  3.379614e-08      ...       5.833151e-08
1052bf702cb099f7  3.944712e-12      ...       4.587294e-07
10543c918a43a8d   2.035780e-08      ...       3.239857e-06
105c9a3453da79c3  5.132968e-07      ...       1.263507e-05
1060688bbf6eac87  3.539738e-09      ...       1.713858e-05
106a247caeabd15a  2.298643e-08      ...       2.986884e-07
106e21606add59f3  1.100202e-07      ...       1.006900e-07
107c38495881b6c9  1.111243e-08      ...       2.451584e-06
108815dd3752ab64  1.371817e-13      ...       1.716095e-09
10943defdd5d5e89  5.326525e-09      ...       2.100118e-05
10a39a78c44ef27c  1.120426e-07      ...       1.692975e-04
10ab70df067bdb4   1.788902e-07      ...       3.711071e-09
10b28e3de3566582  7.317997e-09      ...       6.025696e-07
10b32964331a6cc3  3.951727e-08      ...       5.408505e-05
10b4562e7fa6f668  4.163246e-10      ...       7.947249e-07
10db1c338e1d822f  1.673740e-07      ...       3.309259e-06
10e0c215f5f3084e  8.986198e-08      ...       2.805546e-07
10e95bead8e0b35b  6.520012e-11      ...       2.915028e-07
1100d7b0f24fee88  5.609866e-08      ...       3.358630e-05
11099b321e8c7066  6.118156e-07      ...       8.554547e-05
110df388fd5c50e4  1.731834e-08      ...       3.723453e-06
113520ea0138f76d  2.689990e-09      ...       1.101666e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  1.572207e-17      ...       1.107096e-11
ff3a9fa43f8eab9c  1.312992e-14      ...       9.454205e-08
ff44490e20740a19  1.912019e-08      ...       1.453762e-05
ff481bb029678d5d  4.862642e-08      ...       1.632480e-06
ff4c3570fb7b90d3  3.443657e-06      ...       2.548394e-07
ff4f548d08414709  8.020916e-06      ...       4.918343e-08
ff668377a518ea5f  5.170734e-10      ...       1.199333e-06
ff6a549b2d7a0e76  6.420908e-08      ...       3.228722e-08
ff6ee1b37c8dc1ae  1.604857e-07      ...       1.297163e-05
ff85460d6b853b49  4.677476e-08      ...       4.416861e-08
ff8721b85d1b5a5   3.516485e-12      ...       5.431879e-10
ff8bef7d0de52b31  3.452541e-03      ...       2.796940e-08
ff92504c82c41e0f  1.212175e-09      ...       2.380471e-05
ff9d4b77c124c9f2  1.023232e-09      ...       1.379335e-05
ff9ddf70cb1c2674  9.839931e-09      ...       9.280532e-07
ffaf8c3fe0b1d9b6  1.003135e-12      ...       2.740771e-09
ffb61df4a6734772  4.932638e-22      ...       2.443908e-13
ffb73f95b8721900  9.114825e-13      ...       1.153463e-10
ffb937b55755323e  3.586833e-07      ...       1.938308e-06
ffbcf8b91a8e8ce0  8.663803e-10      ...       2.392345e-07
ffbf4849bde21b0a  1.647541e-13      ...       3.778245e-08
ffc96e053345419d  3.672872e-11      ...       3.368620e-07
ffcb16053099d795  4.970662e-09      ...       3.489774e-07
ffcf745289465074  2.315815e-10      ...       7.122575e-06
ffd1372fe67e65f0  3.025247e-11      ...       3.166707e-12
ffd79eadf642221b  1.190819e-13      ...       1.685278e-08
ffd96986aa333f4d  1.808294e-09      ...       7.949626e-05
ffe54b454396d97c  2.054257e-06      ...       3.402388e-07
ffe7d7db4e4aa37f  1.954962e-02      ...       2.164900e-07
ffed0a4aca0d5457  1.343106e-12      ...       1.341981e-09

[7443 rows x 1103 columns]]
Predict:   0%|          | 0/931 [00:00<?, ?it/s]
Loaded model from epoch 9, step 43,728
14,886 in valid
Predict: 100%|██████████| 931/931 [07:21<00:00,  2.48it/s]
probs:                               0      ...               1102
10023b2cc4ed5f68  1.029401e-10      ...       3.467918e-09
100fbe75ed8fd887  6.591121e-12      ...       1.081128e-08
101b627524a04f19  3.189665e-15      ...       5.040387e-08
10234480c41284c6  4.703811e-10      ...       4.344588e-08
1023b0e2636dcea8  8.141412e-10      ...       3.896905e-08
1039cd6cf85845c   1.773152e-13      ...       5.807459e-09
103a5b3f83fbe88   2.771297e-14      ...       3.391111e-08
10413aaae8d6a9a2  2.935517e-11      ...       5.016654e-07
10423822b93a65ab  2.199924e-09      ...       1.040734e-08
1052bf702cb099f7  2.748813e-12      ...       1.299591e-04

[10 rows x 1103 columns]
[                             0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.728668e-13      ...       1.030075e-10
100fbe75ed8fd887  3.072049e-14      ...       4.280107e-07
101b627524a04f19  2.371050e-10      ...       7.791431e-05
10234480c41284c6  9.136196e-11      ...       2.918332e-08
1023b0e2636dcea8  3.340438e-12      ...       1.591823e-08
1039cd6cf85845c   4.337629e-15      ...       1.039632e-09
103a5b3f83fbe88   1.340217e-08      ...       3.086209e-05
10413aaae8d6a9a2  7.633454e-10      ...       4.874447e-07
10423822b93a65ab  1.558484e-09      ...       5.887860e-07
1052bf702cb099f7  5.377024e-14      ...       1.300073e-04
10543c918a43a8d   5.157139e-09      ...       7.896976e-09
105c9a3453da79c3  4.366324e-10      ...       3.055582e-07
1060688bbf6eac87  1.208084e-09      ...       5.593986e-07
106a247caeabd15a  1.765468e-10      ...       1.186001e-08
106e21606add59f3  2.761865e-11      ...       9.735194e-11
107c38495881b6c9  1.366474e-06      ...       3.426261e-05
108815dd3752ab64  4.891811e-15      ...       5.944660e-08
10943defdd5d5e89  6.925465e-12      ...       6.808719e-06
10a39a78c44ef27c  8.010963e-07      ...       3.032649e-04
10ab70df067bdb4   4.453399e-11      ...       2.154509e-11
10b28e3de3566582  2.334930e-08      ...       4.971076e-06
10b32964331a6cc3  5.610753e-09      ...       1.369045e-06
10b4562e7fa6f668  6.153600e-08      ...       2.608227e-05
10db1c338e1d822f  6.803167e-10      ...       3.939526e-07
10e0c215f5f3084e  2.529129e-16      ...       4.029422e-16
10e95bead8e0b35b  9.851347e-12      ...       6.908680e-07
1100d7b0f24fee88  1.280776e-08      ...       6.198838e-06
11099b321e8c7066  2.435142e-08      ...       1.358786e-06
110df388fd5c50e4  6.032639e-10      ...       4.897266e-07
113520ea0138f76d  1.231125e-09      ...       1.025111e-04
...                        ...      ...                ...
ff2da1f0ed3e3ebe  2.739189e-26      ...       1.377843e-14
ff3a9fa43f8eab9c  2.253703e-14      ...       1.077716e-06
ff44490e20740a19  9.080540e-10      ...       3.801882e-06
ff481bb029678d5d  5.033446e-10      ...       9.471140e-07
ff4c3570fb7b90d3  2.584622e-07      ...       2.476596e-08
ff4f548d08414709  8.068520e-07      ...       5.496110e-11
ff668377a518ea5f  1.695071e-09      ...       7.228905e-06
ff6a549b2d7a0e76  6.122934e-08      ...       1.515677e-06
ff6ee1b37c8dc1ae  7.569311e-12      ...       1.531516e-07
ff85460d6b853b49  1.644709e-09      ...       6.753521e-10
ff8721b85d1b5a5   1.707178e-15      ...       3.052756e-11
ff8bef7d0de52b31  8.941545e-04      ...       7.484565e-10
ff92504c82c41e0f  1.490968e-11      ...       1.039263e-05
ff9d4b77c124c9f2  8.088179e-13      ...       2.885554e-08
ff9ddf70cb1c2674  1.476271e-08      ...       3.213697e-05
ffaf8c3fe0b1d9b6  2.524084e-16      ...       7.134760e-09
ffb61df4a6734772  2.452295e-18      ...       3.635929e-11
ffb73f95b8721900  3.496576e-17      ...       3.272325e-12
ffb937b55755323e  6.148928e-09      ...       2.119681e-07
ffbcf8b91a8e8ce0  3.936214e-10      ...       4.039476e-07
ffbf4849bde21b0a  1.047900e-10      ...       2.959225e-06
ffc96e053345419d  1.336849e-09      ...       1.727152e-07
ffcb16053099d795  9.935106e-10      ...       1.743506e-07
ffcf745289465074  4.772135e-10      ...       7.184802e-06
ffd1372fe67e65f0  9.845096e-24      ...       1.447992e-19
ffd79eadf642221b  7.616045e-12      ...       2.628909e-10
ffd96986aa333f4d  1.433656e-09      ...       1.730341e-05
ffe54b454396d97c  1.886243e-06      ...       1.414021e-06
ffe7d7db4e4aa37f  1.040358e-02      ...       6.017941e-10
ffed0a4aca0d5457  1.113444e-12      ...       4.861438e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.169688e-12      ...       9.107919e-11
100fbe75ed8fd887  2.094033e-09      ...       9.253769e-07
101b627524a04f19  8.167342e-15      ...       2.280261e-06
10234480c41284c6  4.473180e-10      ...       1.535988e-08
1023b0e2636dcea8  6.650800e-11      ...       2.151765e-07
1039cd6cf85845c   4.996251e-16      ...       1.122741e-09
103a5b3f83fbe88   1.916447e-11      ...       4.996276e-07
10413aaae8d6a9a2  4.700306e-09      ...       4.433651e-07
10423822b93a65ab  2.013114e-07      ...       2.491866e-07
1052bf702cb099f7  2.040713e-10      ...       2.565025e-05
10543c918a43a8d   3.331643e-08      ...       8.023857e-07
105c9a3453da79c3  8.417312e-13      ...       1.297822e-06
1060688bbf6eac87  3.050659e-09      ...       7.936198e-06
106a247caeabd15a  1.402277e-13      ...       1.042819e-11
106e21606add59f3  2.030312e-10      ...       1.132089e-09
107c38495881b6c9  2.989174e-08      ...       2.510220e-05
108815dd3752ab64  1.272666e-19      ...       5.041187e-11
10943defdd5d5e89  6.676304e-09      ...       2.253270e-05
10a39a78c44ef27c  1.378666e-07      ...       2.025353e-04
10ab70df067bdb4   1.070075e-11      ...       1.749080e-12
10b28e3de3566582  9.555376e-12      ...       7.372234e-09
10b32964331a6cc3  1.762800e-09      ...       2.154288e-06
10b4562e7fa6f668  4.061142e-08      ...       6.333739e-06
10db1c338e1d822f  6.489576e-09      ...       6.065164e-06
10e0c215f5f3084e  2.604376e-14      ...       3.198323e-10
10e95bead8e0b35b  1.891171e-14      ...       7.382744e-09
1100d7b0f24fee88  3.096013e-08      ...       3.619512e-06
11099b321e8c7066  1.361922e-08      ...       3.388539e-05
110df388fd5c50e4  2.705576e-09      ...       1.869140e-06
113520ea0138f76d  5.520975e-09      ...       1.418945e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  4.393324e-17      ...       2.235492e-10
ff3a9fa43f8eab9c  3.861345e-12      ...       6.479478e-06
ff44490e20740a19  7.186737e-10      ...       1.935095e-07
ff481bb029678d5d  1.325141e-09      ...       1.202223e-07
ff4c3570fb7b90d3  2.790299e-08      ...       1.199169e-09
ff4f548d08414709  8.061184e-07      ...       2.487089e-09
ff668377a518ea5f  9.763571e-09      ...       1.240066e-06
ff6a549b2d7a0e76  7.110477e-07      ...       2.132758e-05
ff6ee1b37c8dc1ae  8.808906e-09      ...       2.684915e-06
ff85460d6b853b49  2.092690e-07      ...       3.347819e-07
ff8721b85d1b5a5   1.739395e-13      ...       1.368489e-10
ff8bef7d0de52b31  7.612832e-05      ...       3.112961e-11
ff92504c82c41e0f  6.241245e-13      ...       3.171968e-07
ff9d4b77c124c9f2  2.353988e-09      ...       5.624191e-08
ff9ddf70cb1c2674  7.063157e-10      ...       3.009483e-06
ffaf8c3fe0b1d9b6  2.541497e-12      ...       6.226065e-10
ffb61df4a6734772  1.097592e-26      ...       3.128824e-19
ffb73f95b8721900  3.289229e-18      ...       1.881861e-12
ffb937b55755323e  3.094228e-07      ...       2.350395e-05
ffbcf8b91a8e8ce0  2.189928e-12      ...       7.289162e-09
ffbf4849bde21b0a  4.982286e-13      ...       4.595518e-07
ffc96e053345419d  8.103861e-11      ...       3.864488e-08
ffcb16053099d795  2.100295e-09      ...       2.070636e-07
ffcf745289465074  1.453044e-10      ...       2.270694e-06
ffd1372fe67e65f0  8.377085e-22      ...       2.415360e-21
ffd79eadf642221b  2.746374e-13      ...       3.338333e-10
ffd96986aa333f4d  7.501828e-10      ...       6.431012e-06
ffe54b454396d97c  4.627787e-07      ...       3.034937e-07
ffe7d7db4e4aa37f  3.006166e-02      ...       3.039038e-08
ffed0a4aca0d5457  6.005101e-13      ...       6.678660e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  1.688872e-12      ...       1.233552e-08
100fbe75ed8fd887  8.501226e-18      ...       2.928678e-13
101b627524a04f19  2.754502e-16      ...       1.895944e-06
10234480c41284c6  6.239672e-10      ...       2.116354e-09
1023b0e2636dcea8  5.640889e-11      ...       3.327427e-09
1039cd6cf85845c   5.316236e-14      ...       1.757189e-11
103a5b3f83fbe88   3.230017e-11      ...       5.635370e-08
10413aaae8d6a9a2  1.347011e-09      ...       2.597413e-08
10423822b93a65ab  2.850066e-08      ...       1.002683e-06
1052bf702cb099f7  1.327797e-14      ...       1.762457e-04
10543c918a43a8d   1.374395e-08      ...       4.425313e-06
105c9a3453da79c3  1.803370e-10      ...       4.606464e-06
1060688bbf6eac87  1.407711e-08      ...       1.245683e-05
106a247caeabd15a  2.377964e-09      ...       1.076588e-06
106e21606add59f3  1.502047e-10      ...       5.570284e-10
107c38495881b6c9  3.697720e-07      ...       2.185369e-06
108815dd3752ab64  2.567115e-16      ...       6.964844e-09
10943defdd5d5e89  3.195673e-13      ...       6.107219e-06
10a39a78c44ef27c  3.378688e-07      ...       8.190519e-05
10ab70df067bdb4   9.783815e-09      ...       1.388995e-09
10b28e3de3566582  8.516890e-12      ...       1.151808e-05
10b32964331a6cc3  5.361789e-09      ...       1.095121e-05
10b4562e7fa6f668  2.074001e-10      ...       1.835770e-05
10db1c338e1d822f  2.169353e-08      ...       4.518313e-06
10e0c215f5f3084e  2.484049e-09      ...       2.299547e-08
10e95bead8e0b35b  7.013370e-12      ...       1.277378e-06
1100d7b0f24fee88  2.016052e-08      ...       1.259897e-05
11099b321e8c7066  1.185949e-07      ...       2.246400e-05
110df388fd5c50e4  5.730570e-09      ...       2.721508e-06
113520ea0138f76d  6.833046e-09      ...       2.664440e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  1.217164e-17      ...       2.189270e-11
ff3a9fa43f8eab9c  7.947099e-16      ...       1.604736e-05
ff44490e20740a19  4.647956e-09      ...       6.015821e-05
ff481bb029678d5d  9.050901e-10      ...       9.972478e-08
ff4c3570fb7b90d3  4.873484e-05      ...       1.685198e-07
ff4f548d08414709  2.159410e-08      ...       4.998338e-13
ff668377a518ea5f  4.068058e-10      ...       1.245732e-06
ff6a549b2d7a0e76  3.591847e-06      ...       6.030097e-07
ff6ee1b37c8dc1ae  2.036120e-08      ...       3.787195e-05
ff85460d6b853b49  3.245934e-09      ...       5.576711e-12
ff8721b85d1b5a5   4.492046e-19      ...       6.564167e-16
ff8bef7d0de52b31  6.228408e-03      ...       2.351988e-09
ff92504c82c41e0f  4.867560e-13      ...       1.299587e-07
ff9d4b77c124c9f2  2.684753e-10      ...       4.635839e-05
ff9ddf70cb1c2674  4.570826e-10      ...       7.221037e-09
ffaf8c3fe0b1d9b6  1.269481e-15      ...       5.569583e-10
ffb61df4a6734772  1.189174e-17      ...       6.532761e-14
ffb73f95b8721900  1.816950e-15      ...       9.251801e-12
ffb937b55755323e  2.318472e-07      ...       1.706806e-05
ffbcf8b91a8e8ce0  9.387164e-12      ...       3.274495e-10
ffbf4849bde21b0a  1.933418e-13      ...       8.629256e-09
ffc96e053345419d  4.242912e-11      ...       1.367808e-07
ffcb16053099d795  7.353135e-11      ...       3.171480e-10
ffcf745289465074  1.550135e-09      ...       6.976936e-05
ffd1372fe67e65f0  7.258660e-22      ...       9.557752e-24
ffd79eadf642221b  8.458958e-13      ...       3.482113e-11
ffd96986aa333f4d  1.167264e-09      ...       2.022440e-05
ffe54b454396d97c  1.774958e-04      ...       1.121146e-06
ffe7d7db4e4aa37f  1.224639e-02      ...       3.347652e-08
ffed0a4aca0d5457  8.636182e-14      ...       2.780588e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.252782e-09      ...       8.373580e-09
100fbe75ed8fd887  1.338250e-13      ...       3.233474e-11
101b627524a04f19  1.567435e-12      ...       1.852006e-05
10234480c41284c6  1.993155e-08      ...       1.661433e-06
1023b0e2636dcea8  4.199389e-10      ...       6.184982e-07
1039cd6cf85845c   1.083505e-10      ...       7.835828e-08
103a5b3f83fbe88   1.772693e-10      ...       2.923270e-06
10413aaae8d6a9a2  8.621997e-09      ...       1.186972e-05
10423822b93a65ab  3.379614e-08      ...       5.833151e-08
1052bf702cb099f7  3.944712e-12      ...       4.587294e-07
10543c918a43a8d   2.035780e-08      ...       3.239857e-06
105c9a3453da79c3  5.132968e-07      ...       1.263507e-05
1060688bbf6eac87  3.539738e-09      ...       1.713858e-05
106a247caeabd15a  2.298643e-08      ...       2.986884e-07
106e21606add59f3  1.100202e-07      ...       1.006900e-07
107c38495881b6c9  1.111243e-08      ...       2.451584e-06
108815dd3752ab64  1.371817e-13      ...       1.716095e-09
10943defdd5d5e89  5.326525e-09      ...       2.100118e-05
10a39a78c44ef27c  1.120426e-07      ...       1.692975e-04
10ab70df067bdb4   1.788902e-07      ...       3.711071e-09
10b28e3de3566582  7.317997e-09      ...       6.025696e-07
10b32964331a6cc3  3.951727e-08      ...       5.408505e-05
10b4562e7fa6f668  4.163246e-10      ...       7.947249e-07
10db1c338e1d822f  1.673740e-07      ...       3.309259e-06
10e0c215f5f3084e  8.986198e-08      ...       2.805546e-07
10e95bead8e0b35b  6.520012e-11      ...       2.915028e-07
1100d7b0f24fee88  5.609866e-08      ...       3.358630e-05
11099b321e8c7066  6.118156e-07      ...       8.554547e-05
110df388fd5c50e4  1.731834e-08      ...       3.723453e-06
113520ea0138f76d  2.689990e-09      ...       1.101666e-05
...                        ...      ...                ...
ff2da1f0ed3e3ebe  1.572207e-17      ...       1.107096e-11
ff3a9fa43f8eab9c  1.312992e-14      ...       9.454205e-08
ff44490e20740a19  1.912019e-08      ...       1.453762e-05
ff481bb029678d5d  4.862642e-08      ...       1.632480e-06
ff4c3570fb7b90d3  3.443657e-06      ...       2.548394e-07
ff4f548d08414709  8.020916e-06      ...       4.918343e-08
ff668377a518ea5f  5.170734e-10      ...       1.199333e-06
ff6a549b2d7a0e76  6.420908e-08      ...       3.228722e-08
ff6ee1b37c8dc1ae  1.604857e-07      ...       1.297163e-05
ff85460d6b853b49  4.677476e-08      ...       4.416861e-08
ff8721b85d1b5a5   3.516485e-12      ...       5.431879e-10
ff8bef7d0de52b31  3.452541e-03      ...       2.796940e-08
ff92504c82c41e0f  1.212175e-09      ...       2.380471e-05
ff9d4b77c124c9f2  1.023232e-09      ...       1.379335e-05
ff9ddf70cb1c2674  9.839931e-09      ...       9.280532e-07
ffaf8c3fe0b1d9b6  1.003135e-12      ...       2.740771e-09
ffb61df4a6734772  4.932638e-22      ...       2.443908e-13
ffb73f95b8721900  9.114825e-13      ...       1.153463e-10
ffb937b55755323e  3.586833e-07      ...       1.938308e-06
ffbcf8b91a8e8ce0  8.663803e-10      ...       2.392345e-07
ffbf4849bde21b0a  1.647541e-13      ...       3.778245e-08
ffc96e053345419d  3.672872e-11      ...       3.368620e-07
ffcb16053099d795  4.970662e-09      ...       3.489774e-07
ffcf745289465074  2.315815e-10      ...       7.122575e-06
ffd1372fe67e65f0  3.025247e-11      ...       3.166707e-12
ffd79eadf642221b  1.190819e-13      ...       1.685278e-08
ffd96986aa333f4d  1.808294e-09      ...       7.949626e-05
ffe54b454396d97c  2.054257e-06      ...       3.402388e-07
ffe7d7db4e4aa37f  1.954962e-02      ...       2.164900e-07
ffed0a4aca0d5457  1.343106e-12      ...       1.341981e-09

[7443 rows x 1103 columns],                              0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  1.029401e-10      ...       3.467918e-09
100fbe75ed8fd887  6.591121e-12      ...       1.081128e-08
101b627524a04f19  3.189665e-15      ...       5.040387e-08
10234480c41284c6  4.703811e-10      ...       4.344588e-08
1023b0e2636dcea8  8.141412e-10      ...       3.896905e-08
1039cd6cf85845c   1.773152e-13      ...       5.807459e-09
103a5b3f83fbe88   2.771297e-14      ...       3.391111e-08
10413aaae8d6a9a2  2.935517e-11      ...       5.016654e-07
10423822b93a65ab  2.199924e-09      ...       1.040734e-08
1052bf702cb099f7  2.748813e-12      ...       1.299591e-04
10543c918a43a8d   6.129017e-09      ...       2.929059e-07
105c9a3453da79c3  1.658606e-12      ...       1.018915e-07
1060688bbf6eac87  4.037242e-10      ...       2.734272e-07
106a247caeabd15a  2.443949e-16      ...       2.004403e-12
106e21606add59f3  3.090021e-12      ...       2.066426e-11
107c38495881b6c9  1.243811e-07      ...       2.405999e-05
108815dd3752ab64  5.193538e-15      ...       8.107733e-09
10943defdd5d5e89  1.917653e-11      ...       2.850862e-06
10a39a78c44ef27c  6.085676e-09      ...       5.650973e-05
10ab70df067bdb4   6.781301e-08      ...       4.733777e-08
10b28e3de3566582  4.620926e-12      ...       1.362957e-05
10b32964331a6cc3  7.551040e-10      ...       4.572292e-06
10b4562e7fa6f668  1.121582e-09      ...       3.020082e-05
10db1c338e1d822f  1.502688e-08      ...       5.820997e-06
10e0c215f5f3084e  1.496959e-13      ...       3.157754e-10
10e95bead8e0b35b  3.655507e-15      ...       7.982150e-09
1100d7b0f24fee88  7.505068e-09      ...       2.831067e-05
11099b321e8c7066  9.138134e-08      ...       5.343625e-05
110df388fd5c50e4  1.698170e-10      ...       8.239407e-08
113520ea0138f76d  2.980629e-10      ...       1.462979e-04
...                        ...      ...                ...
ff2da1f0ed3e3ebe  2.674372e-16      ...       3.093268e-08
ff3a9fa43f8eab9c  1.030053e-18      ...       4.270770e-07
ff44490e20740a19  2.298398e-09      ...       6.233982e-05
ff481bb029678d5d  1.853304e-09      ...       3.766307e-08
ff4c3570fb7b90d3  2.647669e-07      ...       2.453649e-06
ff4f548d08414709  1.944898e-05      ...       3.629014e-08
ff668377a518ea5f  1.400366e-10      ...       5.444425e-07
ff6a549b2d7a0e76  7.036843e-08      ...       1.283168e-06
ff6ee1b37c8dc1ae  1.143443e-09      ...       5.300401e-07
ff85460d6b853b49  4.173673e-08      ...       5.186083e-08
ff8721b85d1b5a5   2.983512e-12      ...       1.172378e-09
ff8bef7d0de52b31  7.082545e-03      ...       1.006537e-08
ff92504c82c41e0f  1.848179e-12      ...       2.678387e-06
ff9d4b77c124c9f2  1.698812e-10      ...       1.100861e-06
ff9ddf70cb1c2674  1.534213e-09      ...       2.167676e-05
ffaf8c3fe0b1d9b6  3.286707e-11      ...       2.089202e-07
ffb61df4a6734772  1.662063e-17      ...       1.311194e-12
ffb73f95b8721900  7.248282e-15      ...       2.688681e-10
ffb937b55755323e  3.002280e-09      ...       6.099413e-06
ffbcf8b91a8e8ce0  1.781189e-09      ...       2.643632e-07
ffbf4849bde21b0a  2.807901e-13      ...       4.725910e-08
ffc96e053345419d  7.480619e-12      ...       3.562980e-08
ffcb16053099d795  4.659476e-10      ...       4.319386e-07
ffcf745289465074  2.791446e-11      ...       3.031918e-06
ffd1372fe67e65f0  8.307694e-16      ...       1.888208e-14
ffd79eadf642221b  5.557727e-22      ...       5.011826e-16
ffd96986aa333f4d  6.426818e-11      ...       1.010563e-05
ffe54b454396d97c  1.915333e-06      ...       3.869752e-07
ffe7d7db4e4aa37f  7.082250e-03      ...       4.749401e-07
ffed0a4aca0d5457  9.037560e-15      ...       2.317638e-09

[7443 rows x 1103 columns]]
                             0      ...               1102
id                                  ...                   
10023b2cc4ed5f68  4.728668e-13      ...       1.030075e-10
100fbe75ed8fd887  3.072049e-14      ...       4.280107e-07
101b627524a04f19  2.371050e-10      ...       7.791431e-05
10234480c41284c6  9.136196e-11      ...       2.918332e-08
1023b0e2636dcea8  3.340438e-12      ...       1.591823e-08

[5 rows x 1103 columns]
id
10023b2cc4ed5f68           223 289 344 369 587 766 1059
100fbe75ed8fd887                               231 1039
101b627524a04f19                            79 784 1037
10234480c41284c6    51 147 480 483 738 776 813 830 1046
1023b0e2636dcea8      147 322 737 776 813 954 1046 1092
Name: attribute_ids, dtype: object
In [13]:
print('Done!')
Done!