diff --git a/sparkflow/HogwildSparkModel.py b/sparkflow/HogwildSparkModel.py index bfc59d6..d4d6d65 100644 --- a/sparkflow/HogwildSparkModel.py +++ b/sparkflow/HogwildSparkModel.py @@ -1,13 +1,13 @@ from flask import Flask, request import six.moves.cPickle as pickle -from sparkflow.ml_util import tensorflow_get_weights, tensorflow_set_weights, handle_features, handle_feed_dict, handle_shuffle +from ml_util import tensorflow_get_weights, tensorflow_set_weights, handle_features, handle_feed_dict, handle_shuffle from google.protobuf import json_format import socket import time -import tensorflow as tf +import tensorflow_core as tf import itertools -from sparkflow.RWLock import RWLock +from RWLock import RWLock from multiprocessing import Process import multiprocessing import uuid @@ -19,7 +19,7 @@ log.setLevel(logging.ERROR) -def get_server_weights(master_url='localhost:5000'): +def get_server_weights(master_url='0.0.0.0:5000'): """ This will get the raw weights, pickle load them, and return. """ @@ -28,7 +28,7 @@ def get_server_weights(master_url='localhost:5000'): return weights -def put_deltas_to_server(delta, master_url='localhost:5000'): +def put_deltas_to_server(delta, master_url='0.0.0.0:5000'): """ This updates the master parameters. We just use simple pickle serialization here. """ @@ -36,7 +36,7 @@ def put_deltas_to_server(delta, master_url='localhost:5000'): def handle_model(data, graph_json, tfInput, tfLabel=None, - master_url='localhost:5000', iters=1000, + master_url='0.0.0.0:5000', iters=1000, mini_batch_size=-1, shuffle=True, mini_stochastic_iters=-1, verbose=0, loss_callback=None): is_supervised = tfLabel is not None @@ -47,7 +47,7 @@ def handle_model(data, graph_json, tfInput, tfLabel=None, new_graph = tf.Graph() with tf.Session(graph=new_graph) as sess: tf.train.import_meta_graph(gd) - loss_variable = tf.get_collection(tf.GraphKeys.LOSSES)[0] + loss_variable = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) sess.run(tf.global_variables_initializer()) trainable_variables = tf.trainable_variables() grads = tf.gradients(loss_variable, trainable_variables) @@ -187,7 +187,7 @@ def start_service(self, metagraph, optimizer, port): new_graph = tf.Graph() with new_graph.as_default(): tf.train.import_meta_graph(metagraph) - loss_variable = tf.get_collection(tf.GraphKeys.LOSSES)[0] + loss_variable = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) trainable_variables = tf.trainable_variables() grads = tf.gradients(loss_variable, trainable_variables) grads = list(zip(grads, trainable_variables)) diff --git a/sparkflow/graph_utils.py b/sparkflow/graph_utils.py index 89c644a..fd16c9a 100644 --- a/sparkflow/graph_utils.py +++ b/sparkflow/graph_utils.py @@ -1,4 +1,4 @@ -import tensorflow as tf +import tensorflow_core as tf from google.protobuf import json_format import json @@ -10,7 +10,7 @@ def build_graph(func): """ first_graph = tf.Graph() with first_graph.as_default() as g: - v = func() + v = func mg = json_format.MessageToJson(tf.train.export_meta_graph()) return mg diff --git a/sparkflow/ml_util.py b/sparkflow/ml_util.py index c1d70d4..dc0bcd1 100644 --- a/sparkflow/ml_util.py +++ b/sparkflow/ml_util.py @@ -1,5 +1,5 @@ import numpy as np -import tensorflow as tf +import tensorflow_core as tf import json from google.protobuf import json_format from pyspark.ml.linalg import Vectors diff --git a/sparkflow/tensorflow_async.py b/sparkflow/tensorflow_async.py index a8c5e3c..2aa55e1 100644 --- a/sparkflow/tensorflow_async.py +++ b/sparkflow/tensorflow_async.py @@ -1,5 +1,5 @@ -import tensorflow as tf -from sparkflow.pipeline_util import PysparkReaderWriter +import tensorflow_core as tf +from pipeline_util import PysparkReaderWriter import numpy as np from pyspark.ml.param import Param, Params, TypeConverters @@ -8,8 +8,8 @@ from pyspark.ml import Model from pyspark.ml.util import Identifiable, MLReadable, MLWritable from pyspark import keyword_only -from sparkflow.HogwildSparkModel import HogwildSparkModel -from sparkflow.ml_util import convert_weights_to_json, predict_func +from HogwildSparkModel import HogwildSparkModel +from ml_util import convert_weights_to_json, predict_func from pyspark import SparkContext import json diff --git a/sparkflow/tensorflow_model_loader.py b/sparkflow/tensorflow_model_loader.py index b9e91fd..0e5f497 100644 --- a/sparkflow/tensorflow_model_loader.py +++ b/sparkflow/tensorflow_model_loader.py @@ -1,5 +1,5 @@ -import tensorflow as tf -from sparkflow.tensorflow_async import SparkAsyncDLModel +import tensorflow_core as tf +from tensorflow_async import SparkAsyncDLModel from google.protobuf import json_format from pyspark.ml.pipeline import PipelineModel import json