Transfer imagenet caltech

args_helper.py

@@ -297,6 +297,12 @@ def parse_arguments(self, jupyter_mode=False):
             default=False,
             help="Boolean flag to indicate whether to use bias"
         )
+        parser.add_argument(
+            "--bias_fc", #_lastlayer",
+            action="store_true",
+            default=False,
+            help="Boolean flag to indicate whether to use bias at the last fc layers"
+        )
         parser.add_argument(
             "--freeze-weights",
             action="store_true",
@@ -878,6 +884,18 @@ def parse_arguments(self, jupyter_mode=False):
             default=0,
             help="Use mixed precision or not"
         )
+        parser.add_argument(
+            "--transfer_learning",
+            type=int,
+            default=0,
+            help="Use transfer learning or not"
+        )
+        parser.add_argument(
+            "--uv_decomp",
+            type=int,
+            default=0,
+            help="Use W=UV decomposition or not"
+        )
         parser.add_argument('--transformer_emsize', type=int, default=200,
                     help='size of word embeddings')
         parser.add_argument('--transformer_nhid', type=int, default=200,
@@ -893,6 +911,7 @@ def parse_arguments(self, jupyter_mode=False):
         parser.add_argument('--transformer_nhead', type=int, default=2,
                     help='the number of heads in the encoder/decoder of the transformer model')
 
+
         parser.add_argument(
             "--only-sanity",
             action="store_true",

caltech_exec_GD.sh

@@ -0,0 +1,62 @@
+# ResNet50_tf (changed the output size)
+
+# Weight training
+python main.py --config configs/training/resnet50/caltech_resnet50_training_1FC.yml #> log_caltech_wt_50epoch_1FC 2>&1
+#python main.py --config configs/training/resnet50/caltech_resnet50_training_2FC.yml #> log_caltech_wt_50epoch_2FC 2>&1
+
+
+
+# 1FC
+:<<BLOCK
+python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_50_1FC.yml > log_caltech_hc_sparsity_50_1FC 2>&1
+python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_5_1FC.yml > log_caltech_hc_sparsity_5_1FC 2>&1
+python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_2_1FC.yml > log_caltech_hc_sparsity_2_1FC 2>&1
+python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_50_1FC.yml > log_caltech_ep_sparsity_50_1FC 2>&1
+python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_5_1FC.yml > log_caltech_ep_sparsity_5_1FC 2>&1
+python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_2_1FC.yml > log_caltech_ep_sparsity_2_1FC 2>&1
+BLOCK
+
+
+# 2FC
+:<<BLOCK
+python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_50_2FC.yml > log_caltech_hc_sparsity_50_2FC 2>&1
+python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_5_2FC.yml > log_caltech_hc_sparsity_5_2FC 2>&1
+python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_2_2FC.yml > log_caltech_hc_sparsity_2_2FC 2>&1
+python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_50_2FC.yml > log_caltech_ep_sparsity_50_2FC 2>&1
+python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_5_2FC.yml > log_caltech_ep_sparsity_5_2FC 2>&1
+python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_2_2FC.yml > log_caltech_ep_sparsity_2_2FC 2>&1
+BLOCK
+
+
+
+
+
+
+
+# ======== OLD
+
+
+
+
+
+
+
+# HC
+#:<<BLOCK
+#python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_50.yml > log_caltech_hc_sparsity_50_UV_1000_bias_real_final 2>&1
+#python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_unconstrained.yml > log_caltech_hc_sparsity_unconstrained_UV_1000_bias_real_final 2>&1
+#python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_5.yml > log_caltech_hc_sparsity_5_2lam6_UV 2>&1
+#python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_2.yml > log_caltech_hc_sparsity_2_5lam6_UV 2>&1
+#python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_0_5.yml > log_caltech_hc_sparsity_0_5_1_5lam5_UV 2>&1
+#BLOCK
+#python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_0_5.yml > log_caltech_hc_sparsity_0_5_2lam5_50epoch_mutli 2>&1
+#python main.py --config configs/hypercube/resnet50/caltech101/caltech101_resnet50_hc_sparsity_0_5_1lam5.yml > log_caltech_hc_sparsity_0_5_1lam5_50epoch_mutli 2>&1
+
+# EP
+#:<<BLOCK
+#python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_50.yml > log_caltech_ep_sparsity_50_UV_1000_bias_real_final 2>&1
+#python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_5.yml > log_caltech_ep_sparsity_5_UV 2>&1
+#python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_2.yml > log_caltech_ep_sparsity_2_UV 2>&1
+#python main.py --config configs/ep/resnet50/caltech101/caltech101_resnet50_ep_sparsity_0_5.yml > log_caltech_ep_sparsity_0_5_UV 2>&1
+#BLOCK
+

configs/sanity/caltech_sanity.yml

@@ -0,0 +1,72 @@
+subfolder: resnet50_caltech_1FC_sanity
+
+# algorithm
+algo: 'hc_iter'
+iter_period: 5
+
+
+# Architecture
+arch: ResNet50
+
+# ===== Dataset ===== #
+dataset: Caltech101
+name: resnet50_caltech101_HC
+transfer_learning: True
+
+# ===== Learning Rate Policy ======== #
+optimizer: adam
+lr: 0.001
+lr_policy: cosine_lr #cosine_lr #constant_lr
+fine_tune_optimizer: adam
+fine_tune_lr: 0.0001
+fine_tune_lr_policy: multistep_lr #cosine_lr #constant_lr
+
+# ===== Network training config ===== #
+epochs: 50 #5
+wd: 0
+momentum: 0.9
+batch_size: 16
+
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: AffineBatchNorm #NonAffineBatchNorm
+freeze_weights: True
+prune_type: BottomK
+#target_sparsity: 2
+# decide if you want to "unflag"
+unflag_before_finetune: True
+init: signed_constant
+score_init: unif #skew #half #bimodal #skew # bern
+scale_fan: False
+
+# ===== Rounding ===== #
+round: naive
+noise: True
+noise_ratio: 0
+
+# ===== Quantization ===== #
+hc_quantized: True
+quantize_threshold: 0.5
+
+# ===== Regularization ===== #
+regularization: L2
+#lmbda: 0.000005
+
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_sanity_checks: True
+only_sanity: True
+sanity_folder: results/resnet50_caltech_HC_sparsity_50_1FC/
+
+#results/resnet50_caltech_HC_sparsity_5_1FC/
+#results/resnet50_caltech_HC_sparsity_2_1FC/ # AWS-4
+

configs/training/resnet50/caltech_resnet50_training.yml

@@ -0,0 +1,51 @@
+subfolder: resnet50_caltech_weight_training
+trial_num: 1
+score_init: 'all_one'
+bias_lastlayer: True
+
+# algorithm
+algo: 'hc_iter'
+weight_training: True
+transfer_learning: True
+
+
+# Architecture
+arch: ResNet50
+
+# ===== Dataset ===== #
+dataset: Caltech101
+name: resnet50_caltech101_training
+
+# ===== Learning Rate Policy ======== #
+optimizer: adam
+lr: 0.0001
+lr_policy: constant_lr #multistep_lr #cosine_lr #constant_lr
+
+# ===== Network training config ===== #
+epochs: 5 #5
+wd: 0
+momentum: 0.9
+batch_size: 16
+
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: AffineBatchNorm #NonAffineBatchNorm
+freeze_weights: True
+prune_rate: -1
+init: kaiming_normal
+scale_fan: True
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_fine_tune: True
+skip_sanity_checks: True
+
+

configs/training/resnet50/caltech_resnet50_training_1FC.yml

@@ -0,0 +1,51 @@
+subfolder: resnet50_caltech_weight_training_1FC
+trial_num: 1
+score_init: 'all_one'
+#bias_fc: True
+
+# algorithm
+algo: 'hc_iter'
+weight_training: True
+transfer_learning: True
+
+
+# Architecture
+arch: ResNet50
+
+# ===== Dataset ===== #
+dataset: Caltech101
+name: resnet50_caltech101_training
+
+# ===== Learning Rate Policy ======== #
+optimizer: adam
+lr: 0.0001
+lr_policy: multistep_lr #cosine_lr #constant_lr
+
+# ===== Network training config ===== #
+epochs: 50 #5
+wd: 0
+momentum: 0.9
+batch_size: 16
+
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: AffineBatchNorm #NonAffineBatchNorm
+freeze_weights: True
+prune_rate: -1
+init: kaiming_normal
+scale_fan: True
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_fine_tune: True
+skip_sanity_checks: True
+
+

configs/training/resnet50/caltech_resnet50_training_2FC.yml

@@ -0,0 +1,52 @@
+subfolder: resnet50_caltech_weight_training_2FC
+trial_num: 1
+score_init: 'all_one'
+#bias_fc: True
+uv_decomp: True
+
+# algorithm
+algo: 'hc_iter'
+weight_training: True
+transfer_learning: True
+
+
+# Architecture
+arch: ResNet50
+
+# ===== Dataset ===== #
+dataset: Caltech101
+name: resnet50_caltech101_training
+
+# ===== Learning Rate Policy ======== #
+optimizer: adam
+lr: 0.0001
+lr_policy: multistep_lr #cosine_lr #constant_lr
+
+# ===== Network training config ===== #
+epochs: 50 #5
+wd: 0
+momentum: 0.9
+batch_size: 16
+
+
+# ===== Sparsity =========== #
+conv_type: SubnetConv
+bn_type: AffineBatchNorm #NonAffineBatchNorm
+freeze_weights: True
+prune_rate: -1
+init: kaiming_normal
+scale_fan: True
+
+# ===== Hardware setup ===== #
+workers: 4
+gpu: 0
+
+
+# ===== Checkpointing ===== #
+checkpoint_at_prune: False
+
+# ==== sanity check ==== #
+skip_fine_tune: True
+skip_sanity_checks: True
+
+

data/__init__.py

@@ -4,3 +4,4 @@
 from data.tinyimagenet import TinyImageNet
 from data.mnist import MNIST
 from data.bigcifar import BigCIFAR10
+from data.caltech101 import Caltech101