apply urllib patch

catalyst_simple.py

@@ -11,6 +11,7 @@
 
 import argparse
 import os
+import urllib
 
 from catalyst.dl import AccuracyCallback
 from catalyst.dl import SupervisedRunner
@@ -25,6 +26,13 @@
 from optuna.integration import CatalystPruningCallback
 
 
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+# This is a temporary fix until torchvision v0.9 is released.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 CLASSES = 10
 
 

chainer/chainer_integration.py

@@ -11,6 +11,8 @@
 
 """
 
+import urllib
+
 import chainer
 import chainer.functions as F
 import chainer.links as L
@@ -21,6 +23,13 @@
 from optuna.trial import TrialState
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 if version.parse(chainer.__version__) < version.parse("4.0.0"):
     raise RuntimeError("Chainer>=4.0.0 is required for this example.")
 

chainer/chainer_simple.py

@@ -8,6 +8,8 @@
 
 """
 
+import urllib
+
 import chainer
 import chainer.functions as F
 import chainer.links as L
@@ -18,6 +20,13 @@
 from optuna.integration import ChainerPruningExtension
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 if version.parse(chainer.__version__) < version.parse("4.0.0"):
     raise RuntimeError("Chainer>=4.0.0 is required for this example.")
 

chainer/chainermn_integration.py

@@ -14,6 +14,7 @@
 """
 
 import sys
+import urllib
 
 import chainer
 import chainer.functions as F
@@ -25,6 +26,13 @@
 from optuna.trial import TrialState
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 N_TRAIN_EXAMPLES = 3000
 N_VALID_EXAMPLES = 1000
 BATCHSIZE = 128

chainer/chainermn_simple.py

@@ -12,6 +12,7 @@
 
 """
 import sys
+import urllib
 
 import chainer
 import chainer.functions as F
@@ -22,6 +23,13 @@
 import optuna
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 N_TRAIN_EXAMPLES = 3000
 N_VALID_EXAMPLES = 1000
 BATCHSIZE = 128

fastai/fastaiv1_simple.py

@@ -16,13 +16,20 @@
 
 import argparse
 from functools import partial
+import urllib
 
 from fastai import vision
 
 import optuna
 from optuna.integration import FastAIV1PruningCallback
 
 
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 BATCHSIZE = 128
 EPOCHS = 10
 

fastai/fastaiv2_simple.py

@@ -15,6 +15,7 @@
 """
 
 import argparse
+import urllib
 
 from fastai.vision.all import accuracy
 from fastai.vision.all import aug_transforms
@@ -28,6 +29,14 @@
 from optuna.integration import FastAIPruningCallback
 
 
+# TODO(crcrpar): Remove the below three lines once fastai gets compatible with torchvision v0.9.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+# This is a temporary fix until torchvision v0.9 is released.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 BATCHSIZE = 128
 EPOCHS = 10
 

haiku_simple.py

@@ -13,6 +13,7 @@
 from typing import Generator
 from typing import Mapping
 from typing import Tuple
+import urllib
 
 import haiku as hk
 import jax
@@ -24,6 +25,13 @@
 import optuna
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 OptState = Any
 Batch = Mapping[str, np.ndarray]
 

keras/keras_integration.py

@@ -14,6 +14,7 @@
     https://github.com/optuna/optuna/blob/master/examples/mlflow/keras_mlflow.py
 
 """
+import urllib
 import warnings
 
 import keras
@@ -27,6 +28,13 @@
 from optuna.trial import TrialState
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 N_TRAIN_EXAMPLES = 3000
 N_VALID_EXAMPLES = 1000
 BATCHSIZE = 128

keras/keras_simple.py

@@ -6,6 +6,7 @@
 Keras. We optimize the filter and kernel size, kernel stride and layer activation.
 
 """
+import urllib
 import warnings
 
 from keras.backend import clear_session
@@ -19,6 +20,13 @@
 import optuna
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 N_TRAIN_EXAMPLES = 3000
 N_VALID_EXAMPLES = 1000
 BATCHSIZE = 128

multi_objective/pytorch_simple.py

@@ -9,6 +9,7 @@
 """
 
 import os
+import urllib
 
 import thop
 import torch
@@ -22,6 +23,13 @@
 import optuna
 
 
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+# This is a temporary fix until torchvision v0.9 is released.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 DEVICE = torch.device("cpu")
 BATCHSIZE = 128
 CLASSES = 10

mxnet/gluon_simple.py

@@ -1,3 +1,5 @@
+import urllib
+
 import mxnet as mx
 from mxnet import autograd
 from mxnet import gluon
@@ -7,6 +9,13 @@
 import optuna
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 CUDA = False
 EPOCHS = 10
 BATCHSIZE = 128

mxnet/mxnet_integration.py

@@ -12,6 +12,7 @@
 """
 
 import logging
+import urllib
 
 import mxnet as mx
 import numpy as np
@@ -21,6 +22,13 @@
 from optuna.trial import TrialState
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 N_TRAIN_EXAMPLES = 3000
 N_TEST_EXAMPLES = 1000
 BATCHSIZE = 128

mxnet/mxnet_simple.py

@@ -9,13 +9,21 @@
 """
 
 import logging
+import urllib
 
 import mxnet as mx
 import numpy as np
 
 import optuna
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 N_TRAIN_EXAMPLES = 3000
 N_VALID_EXAMPLES = 1000
 BATCHSIZE = 128

pytorch/skorch_simple.py

@@ -12,6 +12,7 @@
 """
 
 import argparse
+import urllib
 
 import numpy as np
 from sklearn.datasets import fetch_openml
@@ -26,6 +27,13 @@
 from optuna.integration import SkorchPruningCallback
 
 
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+# This is a temporary fix until torchvision v0.9 is released.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 SUBSET_RATIO = 0.4
 
 mnist = fetch_openml("mnist_784", cache=False)

tensorboard_simple.py

@@ -1,9 +1,18 @@
+import urllib
+
 import tensorflow as tf
 
 import optuna
 from optuna.integration.tensorboard import TensorBoardCallback
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 fashion_mnist = tf.keras.datasets.fashion_mnist
 
 (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()

tensorflow/tensorflow_eager_simple.py

@@ -7,13 +7,22 @@
 
 """
 
+import urllib
+
 from packaging import version
 import tensorflow as tf
 from tensorflow.keras.datasets import mnist
 
 import optuna
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 if version.parse(tf.__version__) < version.parse("2.0.0"):
     raise RuntimeError("tensorflow>=2.0.0 is required for this example.")
 

tensorflow/tensorflow_estimator_integration.py

@@ -13,6 +13,7 @@
 
 import shutil
 import tempfile
+import urllib
 
 import tensorflow as tf
 import tensorflow_datasets as tfds
@@ -21,6 +22,13 @@
 from optuna.trial import TrialState
 
 
+# TODO(crcrpar): Remove the below three lines once everything is ok.
+# Register a global custom opener to avoid HTTP Error 403: Forbidden when downloading MNIST.
+opener = urllib.request.build_opener()
+opener.addheaders = [("User-agent", "Mozilla/5.0")]
+urllib.request.install_opener(opener)
+
+
 MODEL_DIR = tempfile.mkdtemp()
 BATCH_SIZE = 128
 TRAIN_STEPS = 1000