c7fa1d60
Chunk
refractoration st...
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
|
"""
Common utils.
@author: chunk
chunkplus@gmail.com
2014 Dec
"""
__author__ = 'hadoop'
import os, sys
import time
import StringIO
import ConfigParser
|
be12257b
Chunk
data-feat-model f...
|
15
16
|
import numpy as np
|
018ebf56
Chunk
Spark Streaming T...
|
17
|
|
f69baeb6
Chunk
spark streaming ...
|
18
|
class Timer():
|
c7fa1d60
Chunk
refractoration st...
|
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
def __init__(self):
self.__newtime = time.time()
self.__oldtime = self.__newtime
def mark(self):
self.__oldtime = self.__newtime
self.__newtime = time.time()
return self.__newtime - self.__oldtime
def report(self):
print "%-24s%fs" % ("time elapsed:", self.mark())
def ttimer():
newtime = time.time()
while True:
oldtime = newtime
newtime = time.time()
yield newtime - oldtime
def ctimer():
newtime = time.clock()
while True:
oldtime = newtime
newtime = time.clock()
yield newtime - oldtime
def ski2cv(img):
if img.ndim >= 3 and img.shape[2] >= 3:
img[:, :, [0, 2]] = img[:, :, [2, 0]]
return img
def get_env_variable(var_name, default=False):
"""
Get the environment variable or return exception
:param var_name: Environment Variable to lookup
Ref - http://stackoverflow.com/questions/21538859/pycharm-set-environment-variable-for-run-manage-py-task
(c) rh0dium
2015 Jan
"""
try:
return os.environ[var_name]
except KeyError:
import StringIO
import ConfigParser
env_file = os.environ.get('PROJECT_ENV_FILE', "res/.env")
try:
|
018ebf56
Chunk
Spark Streaming T...
|
71
|
config = StringIO.StringIO()
|
f69baeb6
Chunk
spark streaming ...
|
72
|
config.write("[DATA]\n")
|
c7fa1d60
Chunk
refractoration st...
|
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
config.write(open(env_file).read())
config.seek(0, os.SEEK_SET)
cp = ConfigParser.ConfigParser()
cp.readfp(config)
value = dict(cp.items('DATA'))[var_name.lower()]
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
elif value.startswith("'") and value.endswith("'"):
value = value[1:-1]
os.environ.setdefault(var_name, value)
return value
except (KeyError, IOError):
if default is not False:
return default
from django.core.exceptions import ImproperlyConfigured
error_msg = "Either set the env variable '{var}' or place it in your " \
"{env_file} file as '{var} = VALUE'"
raise ImproperlyConfigured(error_msg.format(var=var_name, env_file=env_file))
# Make this unique, and don't share it with anybody.
# e.g. SECRET_KEY = get_env_variable('SECRET_KEY')
def load_env(default=False):
env_file = os.environ.get('PROJECT_ENV_FILE', "res/.env")
try:
config = StringIO.StringIO()
|
018ebf56
Chunk
Spark Streaming T...
|
101
|
config.write("[DATA]\n")
|
f69baeb6
Chunk
spark streaming ...
|
102
|
config.write(open(env_file).read())
|
c7fa1d60
Chunk
refractoration st...
|
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
config.seek(0, os.SEEK_SET)
cp = ConfigParser.ConfigParser()
cp.readfp(config)
for var_name, value in dict(cp.items('DATA')).items():
# print var_name,value
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
elif value.startswith("'") and value.endswith("'"):
value = value[1:-1]
os.environ.setdefault(var_name.upper(), value)
except (KeyError, IOError):
if default is not False:
return default
from django.core.exceptions import ImproperlyConfigured
error_msg = "Either set the env variable '{var}' or place it in your " \
"{env_file} file as '{var} = VALUE'"
raise ImproperlyConfigured(error_msg.format(var='load_env', env_file=env_file))
def bytes2bits(arry_bytes):
"""
:param arry_bytes: 1-D np.unit8 array
:return: 1-D 0/1 array
|
1d19f0e7
Chunk
staged.
|
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
"""
hid_data_bits = [map(int, '{0:08b}'.format(byte)) for byte in arry_bytes]
return np.array(hid_data_bits).ravel()
def bits2bytes(arry_bits):
"""
:param arry_bits: 1-D 0/1 array
:return: 1-D np.unit8 array
"""
str_bits = ''.join(map(str, arry_bits))
arry_bytes = [int(str_bits[i:i + 8], 2) for i in range(0, len(str_bits), 8)]
return np.array(arry_bytes, dtype=np.uint8).ravel()
def test_grammer():
|
be12257b
Chunk
data-feat-model f...
|
143
|
a = 'fsaf'
|
1d19f0e7
Chunk
staged.
|
144
145
|
b = ['dasf', 'dff']
c = 'dgfsfdg'
|
c7fa1d60
Chunk
refractoration st...
|
146
|
# print a + b
|
be12257b
Chunk
data-feat-model f...
|
147
148
149
150
151
152
153
154
|
print [a] + b # ['fsaf', 'dasf', 'dff']
print [a] + [b] # ['fsaf', ['dasf', 'dff']]
print [a] + [c] # ['fsaf', 'dgfsfdg']
if __name__ == '__main__':
timer = Timer()
|
c7fa1d60
Chunk
refractoration st...
|
155
156
157
158
159
160
161
162
163
164
165
166
|
timer.mark()
timer.report()
timer.mark()
time.sleep(1)
# for i in range(1000000):
# print i
timer.report()
# load_env()
# print os.environ
# print os.getenv('SPARK_HOME')
|