#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Sources and sinks.
A Source manages record-oriented data input from a particular kind of source
(e.g. a set of files, a database table, etc.). The reader() method of a source
returns a reader object supporting the iterator protocol; iteration yields
raw records of unprocessed, serialized data.
A Sink manages record-oriented data output to a particular kind of sink
(e.g. a set of files, a database table, etc.). The writer() method of a sink
returns a writer object supporting writing records of serialized data to
the sink.
"""
from collections import namedtuple
import logging
import random
import uuid
from apache_beam import pvalue
from apache_beam import coders
from apache_beam.pvalue import AsIter
from apache_beam.pvalue import AsSingleton
from apache_beam.transforms import core
from apache_beam.transforms import ptransform
from apache_beam.transforms import window
from apache_beam.transforms.display import HasDisplayData
from apache_beam.transforms.display import DisplayDataItem
from apache_beam.utils.windowed_value import WindowedValue
__all__ = ['BoundedSource', 'RangeTracker', 'Read', 'Sink', 'Write', 'Writer']
# Encapsulates information about a bundle of a source generated when method
# BoundedSource.split() is invoked.
# This is a named 4-tuple that has following fields.
# * weight - a number that represents the size of the bundle. This value will
# be used to compare the relative sizes of bundles generated by the
# current source.
# The weight returned here could be specified using a unit of your
# choice (for example, bundles of sizes 100MB, 200MB, and 700MB may
# specify weights 100, 200, 700 or 1, 2, 7) but all bundles of a
# source should specify the weight using the same unit.
# * source - a BoundedSource object for the bundle.
# * start_position - starting position of the bundle
# * stop_position - ending position of the bundle.
#
# Type for start and stop positions are specific to the bounded source and must
# be consistent throughout.
SourceBundle = namedtuple(
'SourceBundle',
'weight source start_position stop_position')
[docs]class BoundedSource(HasDisplayData):
"""A source that reads a finite amount of input records.
This class defines following operations which can be used to read the source
efficiently.
* Size estimation - method ``estimate_size()`` may return an accurate
estimation in bytes for the size of the source.
* Splitting into bundles of a given size - method ``split()`` can be used to
split the source into a set of sub-sources (bundles) based on a desired
bundle size.
* Getting a RangeTracker - method ``get_range_tracker()`` should return a
``RangeTracker`` object for a given position range for the position type
of the records returned by the source.
* Reading the data - method ``read()`` can be used to read data from the
source while respecting the boundaries defined by a given
``RangeTracker``.
A runner will perform reading the source in two steps.
(1) Method ``get_range_tracker()`` will be invoked with start and end
positions to obtain a ``RangeTracker`` for the range of positions the
runner intends to read. Source must define a default initial start and end
position range. These positions must be used if the start and/or end
positions passed to the method ``get_range_tracker()`` are ``None``
(2) Method read() will be invoked with the ``RangeTracker`` obtained in the
previous step.
**Mutability**
A ``BoundedSource`` object should not be mutated while
its methods (for example, ``read()``) are being invoked by a runner. Runner
implementations may invoke methods of ``BoundedSource`` objects through
multi-threaded and/or reentrant execution modes.
"""
[docs] def estimate_size(self):
"""Estimates the size of source in bytes.
An estimate of the total size (in bytes) of the data that would be read
from this source. This estimate is in terms of external storage size,
before performing decompression or other processing.
Returns:
estimated size of the source if the size can be determined, ``None``
otherwise.
"""
raise NotImplementedError
[docs] def split(self, desired_bundle_size, start_position=None, stop_position=None):
"""Splits the source into a set of bundles.
Bundles should be approximately of size ``desired_bundle_size`` bytes.
Args:
desired_bundle_size: the desired size (in bytes) of the bundles returned.
start_position: if specified the given position must be used as the
starting position of the first bundle.
stop_position: if specified the given position must be used as the ending
position of the last bundle.
Returns:
an iterator of objects of type 'SourceBundle' that gives information about
the generated bundles.
"""
raise NotImplementedError
[docs] def get_range_tracker(self, start_position, stop_position):
"""Returns a RangeTracker for a given position range.
Framework may invoke ``read()`` method with the RangeTracker object returned
here to read data from the source.
Args:
start_position: starting position of the range. If 'None' default start
position of the source must be used.
stop_position: ending position of the range. If 'None' default stop
position of the source must be used.
Returns:
a ``RangeTracker`` for the given position range.
"""
raise NotImplementedError
[docs] def read(self, range_tracker):
"""Returns an iterator that reads data from the source.
The returned set of data must respect the boundaries defined by the given
``RangeTracker`` object. For example:
* Returned set of data must be for the range
``[range_tracker.start_position, range_tracker.stop_position)``. Note
that a source may decide to return records that start after
``range_tracker.stop_position``. See documentation in class
``RangeTracker`` for more details. Also, note that framework might
invoke ``range_tracker.try_split()`` to perform dynamic split
operations. range_tracker.stop_position may be updated
dynamically due to successful dynamic split operations.
* Method ``range_tracker.try_split()`` must be invoked for every record
that starts at a split point.
* Method ``range_tracker.record_current_position()`` may be invoked for
records that do not start at split points.
Args:
range_tracker: a ``RangeTracker`` whose boundaries must be respected
when reading data from the source. A runner that reads this
source muss pass a ``RangeTracker`` object that is not
``None``.
Returns:
an iterator of data read by the source.
"""
raise NotImplementedError
[docs] def default_output_coder(self):
"""Coder that should be used for the records returned by the source.
Should be overridden by sources that produce objects that can be encoded
more efficiently than pickling.
"""
return coders.registry.get_coder(object)
[docs]class RangeTracker(object):
"""A thread safe object used by Dataflow source framework.
A Dataflow source is defined using a ''BoundedSource'' and a ''RangeTracker''
pair. A ''RangeTracker'' is used by Dataflow source framework to perform
dynamic work rebalancing of position-based sources.
**Position-based sources**
A position-based source is one where the source can be described by a range
of positions of an ordered type and the records returned by the reader can be
described by positions of the same type.
In case a record occupies a range of positions in the source, the most
important thing about the record is the position where it starts.
Defining the semantics of positions for a source is entirely up to the source
class, however the chosen definitions have to obey certain properties in order
to make it possible to correctly split the source into parts, including
dynamic splitting. Two main aspects need to be defined:
1. How to assign starting positions to records.
2. Which records should be read by a source with a range '[A, B)'.
Moreover, reading a range must be *efficient*, i.e., the performance of
reading a range should not significantly depend on the location of the range.
For example, reading the range [A, B) should not require reading all data
before 'A'.
The sections below explain exactly what properties these definitions must
satisfy, and how to use a ``RangeTracker`` with a properly defined source.
**Properties of position-based sources**
The main requirement for position-based sources is *associativity*: reading
records from '[A, B)' and records from '[B, C)' should give the same
records as reading from '[A, C)', where 'A <= B <= C'. This property
ensures that no matter how a range of positions is split into arbitrarily many
sub-ranges, the total set of records described by them stays the same.
The other important property is how the source's range relates to positions of
records in the source. In many sources each record can be identified by a
unique starting position. In this case:
* All records returned by a source '[A, B)' must have starting positions in
this range.
* All but the last record should end within this range. The last record may or
may not extend past the end of the range.
* Records should not overlap.
Such sources should define "read '[A, B)'" as "read from the first record
starting at or after 'A', up to but not including the first record starting
at or after 'B'".
Some examples of such sources include reading lines or CSV from a text file,
reading keys and values from a BigTable, etc.
The concept of *split points* allows to extend the definitions for dealing
with sources where some records cannot be identified by a unique starting
position.
In all cases, all records returned by a source '[A, B)' must *start* at or
after 'A'.
**Split points**
Some sources may have records that are not directly addressable. For example,
imagine a file format consisting of a sequence of compressed blocks. Each
block can be assigned an offset, but records within the block cannot be
directly addressed without decompressing the block. Let us refer to this
hypothetical format as <i>CBF (Compressed Blocks Format)</i>.
Many such formats can still satisfy the associativity property. For example,
in CBF, reading '[A, B)' can mean "read all the records in all blocks whose
starting offset is in '[A, B)'".
To support such complex formats, we introduce the notion of *split points*. We
say that a record is a split point if there exists a position 'A' such that
the record is the first one to be returned when reading the range
'[A, infinity)'. In CBF, the only split points would be the first records
in each block.
Split points allow us to define the meaning of a record's position and a
source's range in all cases:
* For a record that is at a split point, its position is defined to be the
largest 'A' such that reading a source with the range '[A, infinity)'
returns this record.
* Positions of other records are only required to be non-decreasing.
* Reading the source '[A, B)' must return records starting from the first
split point at or after 'A', up to but not including the first split point
at or after 'B'. In particular, this means that the first record returned
by a source MUST always be a split point.
* Positions of split points must be unique.
As a result, for any decomposition of the full range of the source into
position ranges, the total set of records will be the full set of records in
the source, and each record will be read exactly once.
**Consumed positions**
As the source is being read, and records read from it are being passed to the
downstream transforms in the pipeline, we say that positions in the source are
being *consumed*. When a reader has read a record (or promised to a caller
that a record will be returned), positions up to and including the record's
start position are considered *consumed*.
Dynamic splitting can happen only at *unconsumed* positions. If the reader
just returned a record at offset 42 in a file, dynamic splitting can happen
only at offset 43 or beyond, as otherwise that record could be read twice (by
the current reader and by a reader of the task starting at 43).
"""
SPLIT_POINTS_UNKNOWN = object()
[docs] def start_position(self):
"""Returns the starting position of the current range, inclusive."""
raise NotImplementedError(type(self))
[docs] def stop_position(self):
"""Returns the ending position of the current range, exclusive."""
raise NotImplementedError(type(self))
[docs] def try_claim(self, position): # pylint: disable=unused-argument
"""Atomically determines if a record at a split point is within the range.
This method should be called **if and only if** the record is at a split
point. This method may modify the internal state of the ``RangeTracker`` by
updating the last-consumed position to ``position``.
** Thread safety **
Methods of the class ``RangeTracker`` including this method may get invoked
by different threads, hence must be made thread-safe, e.g. by using a single
lock object.
Args:
position: starting position of a record being read by a source.
Returns:
``True``, if the given position falls within the current range, returns
``False`` otherwise.
"""
raise NotImplementedError
[docs] def set_current_position(self, position):
"""Updates the last-consumed position to the given position.
A source may invoke this method for records that do not start at split
points. This may modify the internal state of the ``RangeTracker``. If the
record starts at a split point, method ``try_claim()`` **must** be invoked
instead of this method.
Args:
position: starting position of a record being read by a source.
"""
raise NotImplementedError
[docs] def position_at_fraction(self, fraction):
"""Returns the position at the given fraction.
Given a fraction within the range [0.0, 1.0) this method will return the
position at the given fraction compared to the position range
[self.start_position, self.stop_position).
** Thread safety **
Methods of the class ``RangeTracker`` including this method may get invoked
by different threads, hence must be made thread-safe, e.g. by using a single
lock object.
Args:
fraction: a float value within the range [0.0, 1.0).
Returns:
a position within the range [self.start_position, self.stop_position).
"""
raise NotImplementedError
[docs] def try_split(self, position):
"""Atomically splits the current range.
Determines a position to split the current range, split_position, based on
the given position. In most cases split_position and position will be the
same.
Splits the current range '[self.start_position, self.stop_position)'
into a "primary" part '[self.start_position, split_position)' and a
"residual" part '[split_position, self.stop_position)', assuming the
current last-consumed position is within
'[self.start_position, split_position)' (i.e., split_position has not been
consumed yet).
If successful, updates the current range to be the primary and returns a
tuple (split_position, split_fraction). split_fraction should be the
fraction of size of range '[self.start_position, split_position)' compared
to the original (before split) range
'[self.start_position, self.stop_position)'.
If the split_position has already been consumed, returns ``None``.
** Thread safety **
Methods of the class ``RangeTracker`` including this method may get invoked
by different threads, hence must be made thread-safe, e.g. by using a single
lock object.
Args:
position: suggested position where the current range should try to
be split at.
Returns:
a tuple containing the split position and split fraction if split is
successful. Returns ``None`` otherwise.
"""
raise NotImplementedError
[docs] def fraction_consumed(self):
"""Returns the approximate fraction of consumed positions in the source.
** Thread safety **
Methods of the class ``RangeTracker`` including this method may get invoked
by different threads, hence must be made thread-safe, e.g. by using a single
lock object.
Returns:
the approximate fraction of positions that have been consumed by
successful 'try_split()' and 'report_current_position()' calls, or
0.0 if no such calls have happened.
"""
raise NotImplementedError
[docs] def split_points(self):
"""Gives the number of split points consumed and remaining.
For a ``RangeTracker`` used by a ``BoundedSource`` (within a
``BoundedSource.read()`` invocation) this method produces a 2-tuple that
gives the number of split points consumed by the ``BoundedSource`` and the
number of split points remaining within the range of the ``RangeTracker``
that has not been consumed by the ``BoundedSource``.
More specifically, given that the position of the current record being read
by ``BoundedSource`` is current_position this method produces a tuple that
consists of
(1) number of split points in the range [self.start_position(),
current_position) without including the split point that is currently being
consumed. This represents the total amount of parallelism in the consumed
part of the source.
(2) number of split points within the range
[current_position, self.stop_position()) including the split point that is
currently being consumed. This represents the total amount of parallelism in
the unconsumed part of the source.
Methods of the class ``RangeTracker`` including this method may get invoked
by different threads, hence must be made thread-safe, e.g. by using a single
lock object.
** General information about consumed and remaining number of split
points returned by this method. **
* Before a source read (``BoundedSource.read()`` invocation) claims the
first split point, number of consumed split points is 0. This condition
holds independent of whether the input is "splittable". A splittable
source is a source that has more than one split point.
* Any source read that has only claimed one split point has 0 consumed
split points since the first split point is the current split point and
is still being processed. This condition holds independent of whether
the input is splittable.
* For an empty source read which never invokes
``RangeTracker.try_claim()``, the consumed number of split points is 0.
This condition holds independent of whether the input is splittable.
* For a source read which has invoked ``RangeTracker.try_claim()`` n
times, the consumed number of split points is n -1.
* If a ``BoundedSource`` sets a callback through function
``set_split_points_unclaimed_callback()``, ``RangeTracker`` can use that
callback when determining remaining number of split points.
* Remaining split points should include the split point that is currently
being consumed by the source read. Hence if the above callback returns
an integer value n, remaining number of split points should be (n + 1).
* After last split point is claimed remaining split points becomes 1,
because this unfinished read itself represents an unfinished split
point.
* After all records of the source has been consumed, remaining number of
split points becomes 0 and consumed number of split points becomes equal
to the total number of split points within the range being read by the
source. This method does not address this condition and will continue to
report number of consumed split points as
("total number of split points" - 1) and number of remaining split
points as 1. A runner that performs the reading of the source can
detect when all records have been consumed and adjust remaining and
consumed number of split points accordingly.
** Examples **
(1) A "perfectly splittable" input which can be read in parallel down to the
individual records.
Consider a perfectly splittable input that consists of 50 split points.
* Before a source read (``BoundedSource.read()`` invocation) claims the
first split point, number of consumed split points is 0 number of
remaining split points is 50.
* After claiming first split point, consumed number of split points is 0
and remaining number of split is 50.
* After claiming split point #30, consumed number of split points is 29
and remaining number of split points is 21.
* After claiming all 50 split points, consumed number of split points is
49 and remaining number of split points is 1.
(2) a "block-compressed" file format such as ``avroio``, in which a block of
records has to be read as a whole, but different blocks can be read in
parallel.
Consider a block compressed input that consists of 5 blocks.
* Before a source read (``BoundedSource.read()`` invocation) claims the
first split point (first block), number of consumed split points is 0
number of remaining split points is 5.
* After claiming first split point, consumed number of split points is 0
and remaining number of split is 5.
* After claiming split point #3, consumed number of split points is 2
and remaining number of split points is 3.
* After claiming all 5 split points, consumed number of split points is
4 and remaining number of split points is 1.
(3) an "unsplittable" input such as a cursor in a database or a gzip
compressed file.
Such an input is considered to have only a single split point. Number of
consumed split points is always 0 and number of remaining split points
is always 1.
By default ``RangeTracker` returns ``RangeTracker.SPLIT_POINTS_UNKNOWN`` for
both consumed and remaining number of split points, which indicates that the
number of split points consumed and remaining is unknown.
Returns:
A pair that gives consumed and remaining number of split points. Consumed
number of split points should be an integer larger than or equal to zero
or ``RangeTracker.SPLIT_POINTS_UNKNOWN``. Remaining number of split points
should be an integer larger than zero or
``RangeTracker.SPLIT_POINTS_UNKNOWN``.
"""
return (RangeTracker.SPLIT_POINTS_UNKNOWN,
RangeTracker.SPLIT_POINTS_UNKNOWN)
[docs] def set_split_points_unclaimed_callback(self, callback):
"""Sets a callback for determining the unclaimed number of split points.
By invoking this function, a ``BoundedSource`` can set a callback function
that may get invoked by the ``RangeTracker`` to determine the number of
unclaimed split points. A split point is unclaimed if
``RangeTracker.try_claim()`` method has not been successfully invoked for
that particular split point. The callback function accepts a single
parameter, a stop position for the BoundedSource (stop_position). If the
record currently being consumed by the ``BoundedSource`` is at position
current_position, callback should return the number of split points within
the range (current_position, stop_position). Note that, this should not
include the split point that is currently being consumed by the source.
This function must be implemented by subclasses before being used.
Args:
callback: a function that takes a single parameter, a stop position,
and returns unclaimed number of split points for the source read
operation that is calling this function. Value returned from
callback should be either an integer larger than or equal to
zero or ``RangeTracker.SPLIT_POINTS_UNKNOWN``.
"""
raise NotImplementedError
[docs]class Sink(HasDisplayData):
"""This class is deprecated, no backwards-compatibility guarantees.
A resource that can be written to using the ``beam.io.Write`` transform.
Here ``beam`` stands for Apache Beam Python code imported in following manner.
``import apache_beam as beam``.
A parallel write to an ``iobase.Sink`` consists of three phases:
1. A sequential *initialization* phase (e.g., creating a temporary output
directory, etc.)
2. A parallel write phase where workers write *bundles* of records
3. A sequential *finalization* phase (e.g., committing the writes, merging
output files, etc.)
Implementing a new sink requires extending two classes.
1. iobase.Sink
``iobase.Sink`` is an immutable logical description of the location/resource
to write to. Depending on the type of sink, it may contain fields such as the
path to an output directory on a filesystem, a database table name,
etc. ``iobase.Sink`` provides methods for performing a write operation to the
sink described by it. To this end, implementors of an extension of
``iobase.Sink`` must implement three methods:
``initialize_write()``, ``open_writer()``, and ``finalize_write()``.
2. iobase.Writer
``iobase.Writer`` is used to write a single bundle of records. An
``iobase.Writer`` defines two methods: ``write()`` which writes a
single record from the bundle and ``close()`` which is called once
at the end of writing a bundle.
See also ``apache_beam.io.filebasedsink.FileBasedSink`` which provides a
simpler API for writing sinks that produce files.
**Execution of the Write transform**
``initialize_write()`` and ``finalize_write()`` are conceptually called once:
at the beginning and end of a ``Write`` transform. However, implementors must
ensure that these methods are *idempotent*, as they may be called multiple
times on different machines in the case of failure/retry or for redundancy.
``initialize_write()`` should perform any initialization that needs to be done
prior to writing to the sink. ``initialize_write()`` may return a result
(let's call this ``init_result``) that contains any parameters it wants to
pass on to its writers about the sink. For example, a sink that writes to a
file system may return an ``init_result`` that contains a dynamically
generated unique directory to which data should be written.
To perform writing of a bundle of elements, Dataflow execution engine will
create an ``iobase.Writer`` using the implementation of
``iobase.Sink.open_writer()``. When invoking ``open_writer()`` execution
engine will provide the ``init_result`` returned by ``initialize_write()``
invocation as well as a *bundle id* (let's call this ``bundle_id``) that is
unique for each invocation of ``open_writer()``.
Execution engine will then invoke ``iobase.Writer.write()`` implementation for
each element that has to be written. Once all elements of a bundle are
written, execution engine will invoke ``iobase.Writer.close()`` implementation
which should return a result (let's call this ``write_result``) that contains
information that encodes the result of the write and, in most cases, some
encoding of the unique bundle id. For example, if each bundle is written to a
unique temporary file, ``close()`` method may return an object that contains
the temporary file name. After writing of all bundles is complete, execution
engine will invoke ``finalize_write()`` implementation. As parameters to this
invocation execution engine will provide ``init_result`` as well as an
iterable of ``write_result``.
The execution of a write transform can be illustrated using following pseudo
code (assume that the outer for loop happens in parallel across many
machines)::
init_result = sink.initialize_write()
write_results = []
for bundle in partition(pcoll):
writer = sink.open_writer(init_result, generate_bundle_id())
for elem in bundle:
writer.write(elem)
write_results.append(writer.close())
sink.finalize_write(init_result, write_results)
**init_result**
Methods of 'iobase.Sink' should agree on the 'init_result' type that will be
returned when initializing the sink. This type can be a client-defined object
or an existing type. The returned type must be picklable using Dataflow coder
``coders.PickleCoder``. Returning an init_result is optional.
**bundle_id**
In order to ensure fault-tolerance, a bundle may be executed multiple times
(e.g., in the event of failure/retry or for redundancy). However, exactly one
of these executions will have its result passed to the
``iobase.Sink.finalize_write()`` method. Each call to
``iobase.Sink.open_writer()`` is passed a unique bundle id when it is called
by the ``WriteImpl`` transform, so even redundant or retried bundles will have
a unique way of identifying their output.
The bundle id should be used to guarantee that a bundle's output is unique.
This uniqueness guarantee is important; if a bundle is to be output to a file,
for example, the name of the file must be unique to avoid conflicts with other
writers. The bundle id should be encoded in the writer result returned by the
writer and subsequently used by the ``finalize_write()`` method to identify
the results of successful writes.
For example, consider the scenario where a Writer writes files containing
serialized records and the ``finalize_write()`` is to merge or rename these
output files. In this case, a writer may use its unique id to name its output
file (to avoid conflicts) and return the name of the file it wrote as its
writer result. The ``finalize_write()`` will then receive an ``Iterable`` of
output file names that it can then merge or rename using some bundle naming
scheme.
**write_result**
``iobase.Writer.close()`` and ``finalize_write()`` implementations must agree
on type of the ``write_result`` object returned when invoking
``iobase.Writer.close()``. This type can be a client-defined object or
an existing type. The returned type must be picklable using Dataflow coder
``coders.PickleCoder``. Returning a ``write_result`` when
``iobase.Writer.close()`` is invoked is optional but if unique
``write_result`` objects are not returned, sink should, guarantee idempotency
when same bundle is written multiple times due to failure/retry or redundancy.
**More information**
For more information on creating new sinks please refer to the official
documentation at
``https://beam.apache.org/documentation/sdks/python-custom-io#creating-sinks``
"""
[docs] def initialize_write(self):
"""Initializes the sink before writing begins.
Invoked before any data is written to the sink.
Please see documentation in ``iobase.Sink`` for an example.
Returns:
An object that contains any sink specific state generated by
initialization. This object will be passed to open_writer() and
finalize_write() methods.
"""
raise NotImplementedError
[docs] def open_writer(self, init_result, uid):
"""Opens a writer for writing a bundle of elements to the sink.
Args:
init_result: the result of initialize_write() invocation.
uid: a unique identifier generated by the system.
Returns:
an ``iobase.Writer`` that can be used to write a bundle of records to the
current sink.
"""
raise NotImplementedError
[docs] def finalize_write(self, init_result, writer_results):
"""Finalizes the sink after all data is written to it.
Given the result of initialization and an iterable of results from bundle
writes, performs finalization after writing and closes the sink. Called
after all bundle writes are complete.
The bundle write results that are passed to finalize are those returned by
bundles that completed successfully. Although bundles may have been run
multiple times (for fault-tolerance), only one writer result will be passed
to finalize for each bundle. An implementation of finalize should perform
clean up of any failed and successfully retried bundles. Note that these
failed bundles will not have their writer result passed to finalize, so
finalize should be capable of locating any temporary/partial output written
by failed bundles.
If all retries of a bundle fails, the whole pipeline will fail *without*
finalize_write() being invoked.
A best practice is to make finalize atomic. If this is impossible given the
semantics of the sink, finalize should be idempotent, as it may be called
multiple times in the case of failure/retry or for redundancy.
Note that the iteration order of the writer results is not guaranteed to be
consistent if finalize is called multiple times.
Args:
init_result: the result of ``initialize_write()`` invocation.
writer_results: an iterable containing results of ``Writer.close()``
invocations. This will only contain results of successful writes, and
will only contain the result of a single successful write for a given
bundle.
"""
raise NotImplementedError
[docs]class Writer(object):
"""This class is deprecated, no backwards-compatibility guarantees.
Writes a bundle of elements from a ``PCollection`` to a sink.
A Writer ``iobase.Writer.write()`` writes and elements to the sink while
``iobase.Writer.close()`` is called after all elements in the bundle have been
written.
See ``iobase.Sink`` for more detailed documentation about the process of
writing to a sink.
"""
[docs] def write(self, value):
"""Writes a value to the sink using the current writer."""
raise NotImplementedError
[docs] def close(self):
"""Closes the current writer.
Please see documentation in ``iobase.Sink`` for an example.
Returns:
An object representing the writes that were performed by the current
writer.
"""
raise NotImplementedError
[docs]class Read(ptransform.PTransform):
"""A transform that reads a PCollection."""
def __init__(self, source):
"""Initializes a Read transform.
Args:
source: Data source to read from.
"""
super(Read, self).__init__()
self.source = source
[docs] def expand(self, pbegin):
assert isinstance(pbegin, pvalue.PBegin)
self.pipeline = pbegin.pipeline
return pvalue.PCollection(self.pipeline)
[docs] def get_windowing(self, unused_inputs):
return core.Windowing(window.GlobalWindows())
def _infer_output_coder(self, input_type=None, input_coder=None):
if isinstance(self.source, BoundedSource):
return self.source.default_output_coder()
else:
return self.source.coder
[docs] def display_data(self):
return {'source': DisplayDataItem(self.source.__class__,
label='Read Source'),
'source_dd': self.source}
[docs]class Write(ptransform.PTransform):
"""A ``PTransform`` that writes to a sink.
A sink should inherit ``iobase.Sink``. Such implementations are
handled using a composite transform that consists of three ``ParDo``s -
(1) a ``ParDo`` performing a global initialization (2) a ``ParDo`` performing
a parallel write and (3) a ``ParDo`` performing a global finalization. In the
case of an empty ``PCollection``, only the global initialization and
finalization will be performed. Currently only batch workflows support custom
sinks.
Example usage::
pcollection | beam.io.Write(MySink())
This returns a ``pvalue.PValue`` object that represents the end of the
Pipeline.
The sink argument may also be a full PTransform, in which case it will be
applied directly. This allows composite sink-like transforms (e.g. a sink
with some pre-processing DoFns) to be used the same as all other sinks.
This transform also supports sinks that inherit ``iobase.NativeSink``. These
are sinks that are implemented natively by the Dataflow service and hence
should not be updated by users. These sinks are processed using a Dataflow
native write transform.
"""
def __init__(self, sink):
"""Initializes a Write transform.
Args:
sink: Data sink to write to.
"""
super(Write, self).__init__()
self.sink = sink
[docs] def display_data(self):
return {'sink': self.sink.__class__,
'sink_dd': self.sink}
[docs] def expand(self, pcoll):
from apache_beam.runners.dataflow.native_io import iobase as dataflow_io
if isinstance(self.sink, dataflow_io.NativeSink):
# A native sink
return pcoll | 'NativeWrite' >> dataflow_io._NativeWrite(self.sink)
elif isinstance(self.sink, Sink):
# A custom sink
return pcoll | WriteImpl(self.sink)
elif isinstance(self.sink, ptransform.PTransform):
# This allows "composite" sinks to be used like non-composite ones.
return pcoll | self.sink
else:
raise ValueError('A sink must inherit iobase.Sink, iobase.NativeSink, '
'or be a PTransform. Received : %r', self.sink)
class WriteImpl(ptransform.PTransform):
"""Implements the writing of custom sinks."""
def __init__(self, sink):
super(WriteImpl, self).__init__()
self.sink = sink
def expand(self, pcoll):
do_once = pcoll.pipeline | 'DoOnce' >> core.Create([None])
init_result_coll = do_once | 'InitializeWrite' >> core.Map(
lambda _, sink: sink.initialize_write(), self.sink)
if getattr(self.sink, 'num_shards', 0):
min_shards = self.sink.num_shards
if min_shards == 1:
keyed_pcoll = pcoll | core.Map(lambda x: (None, x))
else:
keyed_pcoll = pcoll | core.ParDo(_RoundRobinKeyFn(min_shards))
write_result_coll = (keyed_pcoll
| core.WindowInto(window.GlobalWindows())
| core.GroupByKey()
| 'WriteBundles' >> core.ParDo(
_WriteKeyedBundleDoFn(self.sink),
AsSingleton(init_result_coll)))
else:
min_shards = 1
write_result_coll = (pcoll
| 'WriteBundles' >>
core.ParDo(_WriteBundleDoFn(self.sink),
AsSingleton(init_result_coll))
| 'Pair' >> core.Map(lambda x: (None, x))
| core.WindowInto(window.GlobalWindows())
| core.GroupByKey()
| 'Extract' >> core.FlatMap(lambda x: x[1]))
return do_once | 'FinalizeWrite' >> core.FlatMap(
_finalize_write,
self.sink,
AsSingleton(init_result_coll),
AsIter(write_result_coll),
min_shards)
class _WriteBundleDoFn(core.DoFn):
"""A DoFn for writing elements to an iobase.Writer.
Opens a writer at the first element and closes the writer at finish_bundle().
"""
def __init__(self, sink):
self.writer = None
self.sink = sink
def display_data(self):
return {'sink_dd': self.sink}
def process(self, element, init_result):
if self.writer is None:
self.writer = self.sink.open_writer(init_result, str(uuid.uuid4()))
self.writer.write(element)
def finish_bundle(self):
if self.writer is not None:
yield WindowedValue(self.writer.close(), window.MAX_TIMESTAMP,
[window.GlobalWindow()])
class _WriteKeyedBundleDoFn(core.DoFn):
def __init__(self, sink):
self.sink = sink
def display_data(self):
return {'sink_dd': self.sink}
def process(self, element, init_result):
bundle = element
writer = self.sink.open_writer(init_result, str(uuid.uuid4()))
for e in bundle[1]: # values
writer.write(e)
return [window.TimestampedValue(writer.close(), window.MAX_TIMESTAMP)]
def _finalize_write(_, sink, init_result, write_results, min_shards):
write_results = list(write_results)
extra_shards = []
if len(write_results) < min_shards:
logging.debug(
'Creating %s empty shard(s).', min_shards - len(write_results))
for _ in range(min_shards - len(write_results)):
writer = sink.open_writer(init_result, str(uuid.uuid4()))
extra_shards.append(writer.close())
outputs = sink.finalize_write(init_result, write_results + extra_shards)
if outputs:
return (window.TimestampedValue(v, window.MAX_TIMESTAMP) for v in outputs)
class _RoundRobinKeyFn(core.DoFn):
def __init__(self, count):
self.count = count
def start_bundle(self):
self.counter = random.randint(0, self.count - 1)
def process(self, element):
self.counter += 1
if self.counter >= self.count:
self.counter -= self.count
yield self.counter, element


Overview
This example demonstrates how to send the three different types of text messages.
Work flow
Vehicles which are equipped with Garmin devices can received text messages from users of Geotab.
There are three types of text messages which can be sent:
Skill level
Intermediate


Select burger
- Contains Gluten
Please enter some information so that you can order a burger
Ordered Burgers:
Email {{email}}
Housenumber {{housenumber}}
Gender {{gender}}
Paymentmethod {{paymentmethod}}
- {{burgerName}}
Email {{email}}
Housenumber {{housenumber}}
Gender {{gender}}
Paymentmethod {{paymentmethod}}


@(We didn't receive a payment. Try to pay again.)
@{fi}
@{if query.paid === '1'}
@(Thank you, order has been paid.)
@{fi}
@{if query.cod === '1' || model.iscod }
@(Thank you, order has been placed successfully.)
@{fi}
@{if model.isfinished}
@(Finished)
@{fi}
@{if model.status == 'paid'}
@(Order status:)
Processing
@(Order status:)
Processing
@{foreach m in model.statushistory.reverse()}
@{fi}
@{if model.message}
@{m.date.format('@(yyyy-MM-dd HH:mm)')} - @{m.status}
@{end}
@(Message for you:)
@{model.message.replace(/\n/g, '
')}
')}
@(Customer)
@{model.firstname} @{model.lastname}
@(Company)
@{model.company}
@{model.companyid}
@{model.companyvat}
@{model.companyvat}
@(Email address)
@{model.email}
@(Phone number)
@{model.phone}
@(Note)
@{model.note}
@(Pay using your
PayTM Wallet/Credit Card/Debit Card/Net Banking and UPI options.)
@{if (model.product_code == "" || model.product_code == "none") } @{fi}
@(Pay with this
to Avail instant Cash Backs, EMI offers and Bank Offer)
@(You
will be charged Rs.200 now which will be reduced from your final amount during
delivery.
This amount will not be refunded if you cancel the order.)
@{fi}
@{if (model.referalId == null || model.referalId == "") && model.items &&
model.items[0].iscod }
@{fi}
@{ if model.count == 1 }
@{if(!model.ispickup)}
@{fi}
@{fi}
@{if(model.ispickup)}
@{fi}


Past Experience
Pico Digital
Designed and implemented backend shared memory allocation of FPGA objects to speed up processing and startup of CORDOR Modular Set-Top-Box Platform & Headend Broadcasting System.
Implemented FPGA blade specific daemons for TV broadcasting , Database communication between FPGA modules and provider specific access.
Used ZMQ to transfer, receive and transcode packages through UDP and TCP.
Worked on radio watermark detection software for DVP Satellite Receiver.
Design and implemented subprocess multithreaded processing for IP data reception.
Implemented watermark insertion to broadcast signal and watermark detection for low noise radio signal
Usage of C++11, Boost Library, CMake, GTest, ZMQ, git submodules, atlassian git system.
Homepage
Projects


PG_STATISTIC_INDICATOR
Stores information about the number of rows inserted or deleted since the last ANALYZE. The PG_STATISTIC_INDICATOR table is updated frequently following DML operations, so statistics are approximate.
PG_STATISTIC_INDICATOR is visible only to superusers. For more information, see Visibility of Data in System Tables and Views.
Table Columns
| Column Name | Data Type | Description |
|---|---|---|
| stairelid | oid | Table ID |
| stairows | float | Total number of rows in the table. |
| staiins | float | Number of rows inserted since the last ANALYZE. |
| staidels | float | Number of rows deleted or updated since the last ANALYZE. |
Example
The following example returns information for table changes since the last ANALYZE.
select * from pg_statistic_indicator; stairelid | stairows | staiins | staidels ----------+----------+---------+--------- 108271 | 11 | 0 | 0 108275 | 365 | 0 | 0 108278 | 8798 | 0 | 0 108280 | 91865 | 0 |<PHONE_NUMBER>67 | 89981 | 49990 | 9999 108269 | 808 | 606 | 374 108282 | 152220 | 76110 | 248566


I design and deliver bespoke training for professionals, from half-day sessions to long-term projects. I can offer specialised support for those working with young people.
I design and deliver bespoke training for professionals, from half-day sessions to long-term projects. I can offer specialised support for those working with young people.
I create youth-centered, age-appropriate educational experiences for young people aged 11 and up, from hour-long, one-off sessions to ongoing projects in schools, colleges, youth groups, support services and other settings. I have specialist expertise in working with marginalised groups of young people.
I create self-development and learning opportunities, particularly in relation to sex, sexuality and relationships.
I tailor sessions for specific groups, such as student organisations, women’s, flt* or LGBTQ groups. Options range from 2-hour workshops to weekend programmes.
I also offer private sessions for individuals, couples or other relationship constellations, or groups of friends.


---
layout: post
authors: ["Greg Wilson"]
title: Minutes from 2012-12-19
time: "09:00:00 All-Hands Meeting"
date: 2012-12-19
time: "09:00:00"
tags: ["Community"]
---
Our second all-hands meeting was held on Wednesday, December 19, from 12:00-13:10 Eastern time.
Actions
- Dhavide Aruliah
- Co-author bullet-points and cheat sheet for Unix shell
- Co-author bullet-points and cheat sheet for basic Python
- Amy Brown
- Proof-read Code of Conduct
- Add Code of Conduct to bootcamp "how-to"
- Create graphic for "Participant" badge
- Create PDF certificates for "Participant", "Instructor", and "Organizer"
- Add wording to web site to clarify what does/doesn't qualify for use of "Software Carpentry" name and logo
- Add wording to web site asking people to let us know if they're using our material, and requiring them to check with us if they're using our name/logo
- Keep track of who's talking to what sites about bootcamps
- Get explicit retroactive signoff on contributions
- Neil Chue Hong
- Contact OSS-Watch regarding enforcing trademarks for our material
- Matt Davis
- Arrange bootcamp at Cal Poly
- Arrange bootcamp at San Diego State
- matplotlib material
- Ted Hart
- Co-author bullet-points and cheat sheet for regular expressions
- Arrange bootcamp at ESA conf.
- Bernhard Konrad
- Co-author bullet-points and cheat sheet for testing
- Emily Jane McTavish
- Co-author bullet-points and cheat sheet for regular expressions
- Create ticket to arrange bootcamp at U. Kansas
- Geoff Oxberry
- Create ticket to arrange bootcamp at U. Delaware
- Cait Pickens
- Co-author bullet-points and cheat sheet for Git
- Co-author bullet-points and cheat sheet for basic Python
- Karthik Ram
- Co-author bullet-points and cheat sheet for Git
- Joon Ro
- Alex Viana
- Co-author bullet-points and cheat sheet for SQL
- Ben Waugh
- Co-author bullet-points and cheat sheet for testing
- Ethan White
- Co-author bullet-points and cheat sheet for SQL
- Lynne Williams
- Co-author bullet-points and cheat sheet for Unix shell
- Greg Wilson
- Publish Code of Conduct to web site
- Retroactively issue "Participant" badges for 2012 bootcamp attendees
- Issue "Instructor" and "Organizer" badges
- Finish Instructor's Guide by end of Jan 2013
- All
- Check locally to ensure that it's OK to ask bootcamp participants to agree Code of Conduct
- Claim ticket for "Arrange bootcamp at [site]"
- Introduce us to one more potential bootcamp site for 2013
- Volunteer to help with online office hours
- Dhavide Aruliah
- Erik Bray
- Shreyas Cholia
- Matt Davis
- Ross Dickson
- Steve Haddock
- Mike Hansen
- Geoff Oxberry
- Cait Pickens
- Alex Viana
- Ben Waugh
- Ethan White
- Send Greg a picture of yourself in a Software Carpentry t-shirt
Roll Call
- Aron Ahmadia
- Carlos Anderson
- Dhavide Aruliah
- David Ascher
- Azalee Bostroem
- Erik Bray
- Amy Brown
- Shreyas Cholia
- Neil Chue Hong
- Warren Code
- Matt Davis
- Gabriel Devenyi
- Ross Dickson
- Justin Ely
- Steve Haddock
- Ted Hart
- Konrad Hinsen
- Katy Huff
- Trevor King
- Ian Langmore
- Chris Lasher
- Emily Jane McTavish
- Cameron Neylon
- Geoffrey Oxberry
- Cait Pickens
- Karthik Ram
- Anthony Scopatz
- Alex Viana
- Ben Waugh
- Ethan White
- Lynne Williams
- Greg Wilson
Minutes
- Staffing
- Amy Brown from Continuum to help with administration starting Jan 2013.
- Jon Pipitone to help with web site and online tutorials starting Jan 2013.
- Moved to Git
- Git tutorials to be folded together and put into lessons to teach to scientists
- Code of Conduct
- Many sites not sure that we can set a code of conduct for events that happen on campus
- Code of conduct modeled on the PyCon Code of Conduct
- Those who have hosted events/or know of events scheduled at US universitites, please check and let Greg know what your local policy is
- Might apply to overseas schools too (e.g., Saudi Arabia)
- Problems with inappropriate behavious at bootcamps:
- Rare for problems, but nice to let people know expectations in advance
- Criteria for Badges
- Criteria needs to be clear to ensure the instructors are qualified
- Previous instructors will be grandfathered in
- Badges for learners important as part of grant QA process
- Obstacles:
- No way to know what attendees learned
- Not allowed to publish identifying information without consent --> we would need sign-off giving permission to publish names
- Greg: "We could do a participant badge with consent". Also mentioned "Driver's License Test" to come in future
- Will use http://openbadges.org as a badging tool
- Motion passed "Participant Badges" --> will contact 2012's attendees to tell them they can have a record of attendance
- Will postpone assigning skill specific Participant badges until we have specific criteria (to be developed by the spring)
- Amy will be designing PDF for certificate
- Past participants can apply for badges post-attendance
- Use of "Software Carpentry" Name
- If events focus on four general areas okay:
- modular program structure
- task automation
- version control
- testing
- If the event does not fall into one (or more) of these categories, then they cannot call their workshop/bootcamp Software Carpentry or use the Software Carpentry logo
- Not trying to stop people we don't know from using materials, trying to stop people not teaching SWC content areas from using SWC name.
- Greg still needs to speak with a lawyer about international trademark laws
- Ian Mitchell has a draft, but policy still needs legal review
- Greg will circulate draft later
- Trademark only licensed in Canada currently
- Need to have people to check with SWC central before using the name. Even if we can't legally enforce it (Thanks Konrad!)
- Motion passed "use of Software Carpentry"
- Neil Chue Hong will get in touch with OSS-Watch regarding enforcing trademarks for open source software
- Volunteers to write three-line summary of modules for pre-bootcamp circulation as well as cheat sheets for use in bootcamps
- People going to bootcamps not entirely sure what they will be learning
- Need summary of what SWC teaches to include in agenda
- Summary (for before the workshop) + cheat sheet required --> makes sense for pairs to work on both bcause they will have to think about all the same issues.
- Unix shell - Dhavide -- Lynne
- Fernando and I gave out this UNIX cheat sheet at the last workshop:
- It's CC licensed; might be helpful as a starting point at least, but we liked it.
- That cheat sheet is CC BY-SA, and SWC is CC BY, but maybe not a problem if you keep the cheat sheet separate.
- Version control with Git - Cait, Karthik.
- Basic Python --Dhavide, Cait
- Testing Ben, Bernhard (if you want help) (Yes, thanks, will e-mail you.)
- SQL -- Ethan, Alex Viana
- Regular Expressions - Emily Jane, Ted [what portion of bootcamps include regexp?] [I don't know in general- but folks at UT really liked them last week][I like them too, but we didn't cover them] [Do you guys also cover regexp testers? Kodos is super useful for Python. [like regexpal?]I use regexpal as well Cool! --> grep and sed in shell scripting
- Volunteers to write cheat sheets for modules for use in bootcamps
- See prevous point on 3 line summary. The volunteers for each section will do both the 3 line summary and the cheat sheet
- Volunteers to lead "sales" conversation with not-yet-scheduled bootcamps
- Volunteers to make introductions to new sites
- Companies are okay, even though focus is on public sector
- Bootcamps do not have to be in English; welcome moving the material to other host countries
- Amy to keep track of who is talking with which sites
- [katy] Argonne!
- Katy: Other Chicago area school ideas? [katy] other than UChicago, which we're doing... I can't bottom-line more than one more before this summer... but I could help at Northwestern, UIC. I don't really have contacts at those places... but... hm. [cait] I don't have contacts either (undergrad at Lake Forest College, north of the city, but no grad students there). Northwestern could be interesting.
- [Geoff] Livermore? I could do Philadelphia-area sites also, with enough advance notice, since I grew up around there. I went to the University of Delaware, and can talk to them.
- [Emily Jane] University of Kansas
- [Ted] I have a proposal in for ESA 2013 (Ecological society). [Karthik] yay Ted. I'm maxed out on ESA workshops but will help run it if I don't have schedule conflicts. let me know.
- [Geoff] I could do one at the AIChE conference; it's being run in San Francisco, so it's easy for me to add another couple days.[Karthik] I'm in SF and can help.
- [Matt] Cal Poly, San Diego State
- Volunteers for online office hours (Jan-Feb, once a week)
- Online tutorials not well attended last summer
- Will test having individual help to answer that question
- Greg/John will send out a call in the new year
- Volunteers willing to give 1 hour per week:
- Steve Haddock: sounds like fun to try.
- Dhavide Aruliah: I'm willing to give it a go for an hour on Wednesdays...
- Ethan White: Count me in
- Mike Hansen: I'm onboard.
- Ben Waugh: me too, and have some funding to pay grad students to help
- Matt Davis: Definitely in for this.
- Cait Pickens: I'm in
- Geoff Oxberry: I'm in, after I get trained in the study group
- Ross Dickson: volunteers
- Erik Bray
- Alex Viana
- Shreyas Cholia: onboard assuming travel schedule is not too hectic
- Hosts please encourage people to use the service once it goes online
- Goal is to give people enough knowledge to ask their questions on something like StackOverflow
- Lynne to track questions/responses and create FAQ page for webpage
- Volunteers for minor tasks
- Someone to volunteer to write numpy/matplotlib (#72) [Joon Ro just developed some material as well] [I thought Matt had a start on this?]
- Developers Certificate of Origin/Signed-off-by for Git(Hub) commits. https://github.com/wking/signed-off-by/blob/master/Documentation/SubmittingPatches
- Need retroactive clearance for open source for content developers
- Applying to Sloan Foundation to have someone work FT for SWC for a year
- New teacher materials? - Instructor's Guide Partially Exists
- Hopefully by end of January with John's help
- Next round of study group starts off in January. Let Greg know if you would like to participate
- Big picture question (maybe folded into discussion of long-term assessment): Are two-day workshops the best model for making a difference?
- We have done a couple 2-week class (and official univeristy quarter-long).
- The two-day format seems good at telling you what you should learn, but not necessarily teaching it (with implications for curriculum).
- I like week long formats, personally [Anthony]
- I'm a fan of 2 day workshops, long enough to get some real material in, but able to fit into busy schedules [Emily Jane]
- Greg's Last Request: Send to Greg a nice hi-res photo of yourself in the SWC T-Shirt.


|
Panda3D
|
This is the base class for all three-component vectors and points. More...
Public Types | |
| enum | { num_components = 4, is_int = 0 } |
Public Member Functions | |
| object | __getattr__ (str attr_name) |
| __init__ () | |
| __init__ (const LPoint3f point) | |
| Constructs an LVecBase4 from an LPoint3. More... | |
| __init__ (const LVecBase3f copy, float w) | |
| __init__ (const LVecBase4f) | |
| __init__ (const LVector3f vector) | |
| Constructs an LVecBase4 from an LVector3. More... | |
| __init__ (const UnalignedLVecBase4f copy) | |
| __init__ (float fill_value) | |
| __init__ (float x, float y, float z, float w) | |
| object | __ipow__ (float exponent) |
| LVecBase4f | __pow__ (float exponent) |
| object | __reduce__ () |
| str | __repr__ () |
| int | __setattr__ (str attr_name, object assign) |
| int | addHash (int hash) |
| Adds the vector into the running hash. More... | |
| int | addHash (int hash, float threshold) |
| Adds the vector into the running hash. More... | |
| addToCell (int i, float value) | |
| addW (float value) | |
| addX (float value) | |
| addY (float value) | |
| addZ (float value) | |
| bool | almostEqual (const LVecBase4f other) |
| Returns true if two vectors are memberwise equal within a default tolerance based on the numeric type. More... | |
| bool | almostEqual (const LVecBase4f other, float threshold) |
| Returns true if two vectors are memberwise equal within a specified tolerance. More... | |
| int | compareTo (const LVecBase4f other) |
| This flavor of compare_to uses a default threshold value based on the numeric type. More... | |
| int | compareTo (const LVecBase4f other, float threshold) |
| Sorts vectors lexicographically, componentwise. More... | |
| componentwiseMult (const LVecBase4f other) | |
| float | dot (const LVecBase4f other) |
| fill (float fill_value) | |
| Sets each element of the vector to the indicated fill_value. More... | |
| LVecBase4f | fmax (const LVecBase4f other) |
| LVecBase4f | fmin (const LVecBase4f other) |
| generateHash (ChecksumHashGenerator hashgen) | |
| Adds the vector to the indicated hash generator. More... | |
| generateHash (ChecksumHashGenerator hashgen, float threshold) | |
| Adds the vector to the indicated hash generator. More... | |
| float | getCell (int i) |
| getData () | |
| Returns the address of the first of the four data elements in the vector. More... | |
| int | getHash () |
| Returns a suitable hash for phash_map. More... | |
| int | getHash (float threshold) |
| Returns a suitable hash for phash_map. More... | |
| float | getW () |
| float | getX () |
| LVecBase2f | getXy () |
| Returns the x and y component of this vector. More... | |
| LVecBase3f | getXyz () |
| Returns the x, y and z component of this vector. More... | |
| float | getY () |
| float | getZ () |
| bool | isNan () |
| Returns true if any component of the vector is not-a-number, false otherwise. More... | |
| float | length () |
| Returns the length of the vector, by the Pythagorean theorem. More... | |
| float | lengthSquared () |
| Returns the square of the vector's length, cheap and easy. More... | |
| bool | normalize () |
| Normalizes the vector in place. More... | |
| LVecBase4f | normalized () |
| Normalizes the vector and returns the normalized vector as a copy. More... | |
| operator new (int size) | |
| bool | operator!= (const LVecBase4f other) |
| LVecBase4f | operator* (float scalar) |
| LVecBase4f | operator*= (float scalar) |
| LVecBase4f | operator+ (const LVecBase4f other) |
| LVecBase4f | operator+= (const LVecBase4f other) |
| LVecBase4f | operator- () |
| LVecBase4f | operator- (const LVecBase4f other) |
| LVecBase4f | operator-= (const LVecBase4f other) |
| LVecBase4f | operator/ (float scalar) |
| LVecBase4f | operator/= (float scalar) |
| bool | operator< (const LVecBase4f other) |
| LVecBase4f | operator= (const LVecBase4f copy) |
| LVecBase4f | operator= (float fill_value) |
| bool | operator== (const LVecBase4f other) |
| float | operator[] (int i) |
| operator[] (int i, float assign_val) | |
| output (Ostream out) | |
| LVecBase4f | project (const LVecBase4f onto) |
| Returns a new vector representing the projection of this vector onto another one. More... | |
| readDatagram (DatagramIterator source) | |
| Reads the vector from the Datagram using get_stdfloat(). More... | |
| readDatagramFixed (DatagramIterator source) | |
| Reads the vector from the Datagram using get_float32() or get_float64(). More... | |
| set (float x, float y, float z, float w) | |
| setCell (int i, float value) | |
| setW (float value) | |
| setX (float value) | |
| setY (float value) | |
| setZ (float value) | |
| writeDatagram (Datagram destination) | |
| Writes the vector to the Datagram using add_stdfloat(). More... | |
| writeDatagramFixed (Datagram destination) | |
| Writes the vector to the Datagram using add_float32() or add_float64(), depending on the type of floats in the vector, regardless of the setting of Datagram::set_stdfloat_double(). More... | |
Static Public Member Functions | |
| static TypeHandle | getClassType () |
| static int | getNumComponents () |
| static int | size () |
| static const LVecBase4f | unitW () |
| Returns a unit W vector. More... | |
| static const LVecBase4f | unitX () |
| Returns a unit X vector. More... | |
| static const LVecBase4f | unitY () |
| Returns a unit Y vector. More... | |
| static const LVecBase4f | unitZ () |
| Returns a unit Z vector. More... | |
| static const LVecBase4f | zero () |
| Returns a zero-length vector. More... | |
Public Attributes | |
| float | x |
| LVecBase2f | xy |
| Returns the x and y component of this vector. More... | |
| LVecBase3f | xyz |
| Returns the x, y and z component of this vector. More... | |
| float | y |
| float | z |
This is the base class for all three-component vectors and points.
| object __getattr__ | ( | str | attr_name | ) |
| __init__ | ( | ) |
| __init__ | ( | const LPoint3f | point | ) |
Constructs an LVecBase4 from an LPoint3.
The w coordinate is set to 1.0.
| __init__ | ( | const LVecBase3f | copy, |
| float | w | ||
| ) |
| __init__ | ( | const | LVecBase4f | ) |
| __init__ | ( | const LVector3f | vector | ) |
Constructs an LVecBase4 from an LVector3.
The w coordinate is set to 0.0.
| __init__ | ( | const UnalignedLVecBase4f | copy | ) |
| __init__ | ( | float | fill_value | ) |
| __init__ | ( | float | x, |
| float | y, | ||
| float | z, | ||
| float | w | ||
| ) |
| object __ipow__ | ( | float | exponent | ) |
| LVecBase4f __pow__ | ( | float | exponent | ) |
| object __reduce__ | ( | ) |
| str __repr__ | ( | ) |
| int __setattr__ | ( | str | attr_name, |
| object | assign | ||
| ) |
| int addHash | ( | int | hash | ) |
Adds the vector into the running hash.
| int addHash | ( | int | hash, |
| float | threshold | ||
| ) |
Adds the vector into the running hash.
| addToCell | ( | int | i, |
| float | value | ||
| ) |
| addW | ( | float | value | ) |
| addX | ( | float | value | ) |
| addY | ( | float | value | ) |
| addZ | ( | float | value | ) |
| bool almostEqual | ( | const LVecBase4f | other | ) |
Returns true if two vectors are memberwise equal within a default tolerance based on the numeric type.
| bool almostEqual | ( | const LVecBase4f | other, |
| float | threshold | ||
| ) |
Returns true if two vectors are memberwise equal within a specified tolerance.
| int compareTo | ( | const LVecBase4f | other | ) |
This flavor of compare_to uses a default threshold value based on the numeric type.
| int compareTo | ( | const LVecBase4f | other, |
| float | threshold | ||
| ) |
Sorts vectors lexicographically, componentwise.
Returns a number less than 0 if this vector sorts before the other one, greater than zero if it sorts after, 0 if they are equivalent (within the indicated tolerance).
| componentwiseMult | ( | const LVecBase4f | other | ) |
| float dot | ( | const LVecBase4f | other | ) |
| fill | ( | float | fill_value | ) |
Sets each element of the vector to the indicated fill_value.
This is particularly useful for initializing to zero.
| LVecBase4f fmax | ( | const LVecBase4f | other | ) |
| LVecBase4f fmin | ( | const LVecBase4f | other | ) |
| generateHash | ( | ChecksumHashGenerator | hashgen | ) |
Adds the vector to the indicated hash generator.
| generateHash | ( | ChecksumHashGenerator | hashgen, |
| float | threshold | ||
| ) |
Adds the vector to the indicated hash generator.
| float getCell | ( | int | i | ) |
|
static |
| getData | ( | ) |
Returns the address of the first of the four data elements in the vector.
The remaining elements occupy the next positions consecutively in memory.
| int getHash | ( | ) |
Returns a suitable hash for phash_map.
| int getHash | ( | float | threshold | ) |
Returns a suitable hash for phash_map.
|
static |
| float getW | ( | ) |
| float getX | ( | ) |
| LVecBase2f getXy | ( | ) |
Returns the x and y component of this vector.
| LVecBase3f getXyz | ( | ) |
Returns the x, y and z component of this vector.
| float getY | ( | ) |
| float getZ | ( | ) |
| bool isNan | ( | ) |
Returns true if any component of the vector is not-a-number, false otherwise.
| float length | ( | ) |
Returns the length of the vector, by the Pythagorean theorem.
| float lengthSquared | ( | ) |
Returns the square of the vector's length, cheap and easy.
| bool normalize | ( | ) |
Normalizes the vector in place.
Returns true if the vector was normalized, false if it was a zero-length vector.
| LVecBase4f normalized | ( | ) |
Normalizes the vector and returns the normalized vector as a copy.
If the vector was a zero-length vector, a zero length vector will be returned.
| operator new | ( | int | size | ) |
| bool operator!= | ( | const LVecBase4f | other | ) |
| LVecBase4f operator* | ( | float | scalar | ) |
| LVecBase4f operator*= | ( | float | scalar | ) |
| LVecBase4f operator+ | ( | const LVecBase4f | other | ) |
| LVecBase4f operator+= | ( | const LVecBase4f | other | ) |
| LVecBase4f operator- | ( | ) |
| LVecBase4f operator- | ( | const LVecBase4f | other | ) |
| LVecBase4f operator-= | ( | const LVecBase4f | other | ) |
| LVecBase4f operator/ | ( | float | scalar | ) |
| LVecBase4f operator/= | ( | float | scalar | ) |
| bool operator< | ( | const LVecBase4f | other | ) |
| LVecBase4f operator= | ( | const LVecBase4f | copy | ) |
| LVecBase4f operator= | ( | float | fill_value | ) |
| bool operator== | ( | const LVecBase4f | other | ) |
| float operator[] | ( | int | i | ) |
| operator[] | ( | int | i, |
| float | assign_val | ||
| ) |
| output | ( | Ostream | out | ) |
| LVecBase4f project | ( | const LVecBase4f | onto | ) |
Returns a new vector representing the projection of this vector onto another one.
The resulting vector will be a scalar multiple of onto.
| readDatagram | ( | DatagramIterator | source | ) |
Reads the vector from the Datagram using get_stdfloat().
| readDatagramFixed | ( | DatagramIterator | source | ) |
Reads the vector from the Datagram using get_float32() or get_float64().
See write_datagram_fixed().
| set | ( | float | x, |
| float | y, | ||
| float | z, | ||
| float | w | ||
| ) |
| setCell | ( | int | i, |
| float | value | ||
| ) |
| setW | ( | float | value | ) |
| setX | ( | float | value | ) |
| setY | ( | float | value | ) |
| setZ | ( | float | value | ) |
|
static |
|
static |
Returns a unit W vector.
|
static |
Returns a unit X vector.
|
static |
Returns a unit Y vector.
|
static |
Returns a unit Z vector.
| writeDatagram | ( | Datagram | destination | ) |
Writes the vector to the Datagram using add_stdfloat().
This is appropriate when you want to write the vector using the standard width setting, especially when you are writing a bam file.
| writeDatagramFixed | ( | Datagram | destination | ) |
Writes the vector to the Datagram using add_float32() or add_float64(), depending on the type of floats in the vector, regardless of the setting of Datagram::set_stdfloat_double().
This is appropriate when you want to write a fixed-width value to the datagram, especially when you are not writing a bam file.
|
static |
Returns a zero-length vector.
| float x |
| LVecBase2f xy |
Returns the x and y component of this vector.
| LVecBase3f xyz |
Returns the x, y and z component of this vector.
| float y |
| float z |


--- layout: archive title: Zulip Chat Archive permalink: /stream/113489-new-members/topic/inductive.20type.20with.20anonymous.20constructors.html ---
I want to translate the following English sentence into Lean:
"An extended bool is either a value called extra or a boolean value".
Normally I would do it by typing
inductive extended_bool : Type | extra : extended_bool | boolean : bool → extended_bool
However, this means that when working with extended bool, I always need to destruct it and include the boolean constructor for the boolean case. Can I save myself some work by having an "anonymous" constructor? Is this considered bad coding practice?
That looks like option bool
to me
Anonymous constructors unfortunately only work for types with one constructor
But even with option, don't I have to use some
and none
? I wanted some way to say "if it is not extra, then it must be boolean, so I am not going to write it."
Another example might be bool ⊕ nat
. Is there a way for me to define functions on this type without constantly making references to sum.inl
and sum.inr
? As in, can Lean infer the definition of the function simply from the type of the arguments?
You can define a coercion from bool to extended_bool
def ebool := option bool instance : has_coe bool ebool := ⟨λ b, some b⟩ variable f : ebool → ebool #check f tt #check f none
Is that what you want?
In case this is not clear, option bool
without wrapping it already gives you that.
option
actually already has such a coercion
(oops, patrick already said this)
This will work in cases like bool ⊕ nat
, with type based dispatch, but it won't work in patterns (i.e. pattern matching requires explicit inl
and inr
). But I don't think it is good style, and the option coe is rarely used in mathlib
@Patrick Massot yes, that is what I was looking for. That's a cool trick with has_coe
.
@Mario Carneiro Yes, I can see how this can lead to smelly code. I'll try and avoid this kind of usage in practice.
For the case of bool ⊕ nat
, do I still need explicit coercion? I am not sure I understand what you mean by type-based-dispatch. So far I have:
def ebool := bool ⊕ nat def f : ebool → nat -- throws an error | tt := 1 | ff := 0 | n := n
How can I define that f
properly?
like I said, it won't work in patterns
def ebool := bool ⊕ nat instance coe_l : has_coe bool ebool := ⟨sum.inl⟩ instance coe_r : has_coe nat ebool := ⟨sum.inr⟩ example : ebool := tt example : ebool := ff example (n : nat) : ebool := n
Oh, it looks like this works too:
def f : ebool → nat | tt := 1 | ff := 0 | (n : ℕ) := n
Oh, that's just perfect! Thanks. :+1:
{% endraw %}Last updated: Apr 24 2020 at 19:02 UTC


US Navy carrier transits Strait of Hormuz after deployment
by The Associated PressDUBAI, United Arab Emirates — A U.S. aircraft carrier ordered by the White House to rapidly deploy to the Mideast over a perceived threat from Iran has transited the Strait of Hormuz for the first time since its deployment.
The U.S. Navy says the USS Abraham Lincoln transited the strait on Tuesday, making its way to the Persian Gulf. The carrier left Norfolk, VA, in April and was diverted to the Middle East in May, but it had remained in the Arabian Sea, avoiding passage through the strait that borders Iran.
American aircraft carriers have for decades sailed through the international oil shipping route in what the U.S. describes as “defensive” operations aimed at keeping the strait open.
The Trump administration deployed the Lincoln to the Persian Gulf amid a spike in tensions with Iran.
The Associated Press


QuickVec C++ is a modern C++ approach to explicit cross-platform SIMD vectorization. It enables developers to access the power of the hardware SIMD feature sets using a common set of accelerated functions.
Efficiency and performance are important to many high performance applications. QuickVec C++ will be implemented with a focus on performance and efficiency. It will expose the power of the SIMD instructions available on the hardware on each system without forcing the user to duplicate any work.
It is important to developers to work with well designed systems that follow the paradigms that they know and understand. The QuickVec C++ library will attempt to match the patterns presented by the C++ standard library and STL. This will ease the development process greatly.
When developing applications for multiple platforms, it not only inconvenient for developers to use multiple SIMD intrinsic sets, but also inefficient and time-consuming. The QuickVec C++ library will maintain its performance while being fully cross-platform. This means that functions written using the library should run on any of the supported platforms with no extra effort expended.
The major challenge of this project was bringing all of the targets together in one library. The target instruction sets include SSE (versions 1-4.2), AVX (1,2, and 512-bit), and ARM Neon. There are many differing sets of intrinsics for different instruction feature sets and platforms. A large part of the project is determining which features are available for a processor at runtime and executing with little to no overhead in comparison to a sequential implementation if the SIMD instructions are not available. The second challenge is getting results with little to no difference in speed with a similarly written SIMD intrinsics implementation.
C++ is the current standard language for high-performance cross-platform application development. It is also capable of achieving different levels of abstraction that are not available in other languages. Also, base intrinsics are available onall of the target platforms in C or C++ as a base for implementations.
| Week | Task | Done |
|---|---|---|
| April 3 - April 10 | Find Commonalities and Design Interfaces | |
| April 10 - April 17 | Implement sequential and SSE | |
| April 17 - April 24 | Simple instructions for all SSE versions, AVX, and Neon. Create android project to test ARM Neon implementation. Expand test bench for all versions. | |
| April 24 - May 1 | Automatic detection of extensions. Compile-time detection or specification of extensions. Create interface for efficient loads and stores. Add integer type interface for all versions. | |
| May 1 - May 8 | Add compare and masking operators. Add shuffles and shifts. Add dynamically sized array type. Create performance test (QuickVec, Yeppp, intrinsics). | |
| May 8 - May 11 | Prepare for final presentation. |
A test bench using visual studio unit tests is capable of testing the operators for consistency against the sequential implementation of each operator. The test bench tests all implemented methods.
The sequential implementation is designed to be the base class of all further specialized classes (such as SSE or AVX). The implementation does a compile time unroll of whatever operator is being applied across all elements of the vector. This is done using a template meta-programming technique to approximate a compile-time for each. The reason for this is to guarantee the lack of branching when operating on all vector elements. The hope for this is that if a specific platform does not properly report its use of SSE or AVX, that ILP will work smoothly for the set of instructions.
The SSE (all versions) implementations for 128-bit or 4-wide single precision floating point vectors and 32-bit integer vectors are complete. Also, the AVX implementations of 8-wide vectors for 32-bit floats and 32-bit signed integers are complete. These includes common arithmetic, loads, stores, and comparisons.
This is the result of doing a mandelbrot set calculation with a maximum iteration per pixel of 100 times and a resolution of 2048x2048. Each code example was run 100 times to collect the data seen in the chart. The code was compiled using Microsoft Visual Studio with Full Optimization (/Ox) enabled. It should be noted that this means the sequentially written code has been auto-vectorized by the compiler.
The chart to the right shows the speedup relative to the compiler optimized sequential implementation. As shown by the chart, the QuickVec code is as fast as the code written using AVX intrinsics directly. Below, it can be seen from the code example that the same code written with intrinsics becomes harder to understand. However, the code utilizing QuickVec maintains speed, but does so in a way that doesn't obfuscate the intention of the code.
float oneOverRes = 1.0f / RESOLUTION;
for (int iy = 0; iy < RESOLUTION; iy++) {
float y = iy *oneOverRes;
for (int ix = 0; ix < RESOLUTION; ix++) {
float x = ix * oneOverRes;
float z = 0.0f;
float zi = 0.0f;
bool finished = false;
int i;
for (i = 0; i < ITERATIONS; i++) {
z = (z*z) + (zi*zi) + x;
zi = 2.0f*z*zi + y;
if (((z*z) + (zi*zi)) < 4.0f) {
finished = true;
break;
}
}
results[ix + (iy * RESOLUTION)] = i;
}
}
using float_vec = QuickVec::float8_avx;
float_vec increment(0, 1, 2, 3, 4, 5, 6, 7);
float_vec one = 1.0f; //Single elements can be used to construct
float_vec four = 4.0f;
float_vec oneOverRes = 1.0f / RESOLUTION;
for (int iy = 0; iy < RESOLUTION; iy++) {
float_vec y = iy * oneOverRes;
for (int ix = 0; ix < RESOLUTION; ix += float_vec::size) {
float_vec x = (increment + ix) * oneOverRes;
float_vec z, zi, vals; // Construction initializes to 0
for (int i = 0; i < ITERATIONS; i++) {
z = (z*z) + (zi*zi) + x;
// Single elements can be used with operations
zi = 2.0f*z*zi + y;
float_vec::bool_t finished = (((z*z) + (zi*zi)) < four);
vals.if_not_set(finished, vals + one);
if (finished.all())
break;
}
vals.store(&results[ix + (iy * RESOLUTION)]);
}
}
alignas(32) float incr[] = { 0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f };
__m256 increment = _mm256_load_ps(incr);
__m256 one = _mm256_set1_ps(1.0f);
__m256 four = _mm256_set1_ps(4.0f);
__m256 oneOverRes = _mm256_set1_ps(1.0f / RESOLUTION);
for (int iy = 0; iy < RESOLUTION; iy++) {
__m256 y = _mm256_mul_ps(_mm256_set1_ps(iy), oneOverRes);
for (int ix = 0; ix < RESOLUTION; ix += 8) {
__m256 x = _mm256_mul_ps(_mm256_add_ps(increment, _mm256_set1_ps(ix)), oneOverRes);
__m256 z = _mm256_setzero_ps();
__m256 zi = _mm256_setzero_ps();
__m256 vals = _mm256_setzero_ps();
for (int i = 0; i < ITERATIONS; i++) {
z = _mm256_add_ps(_mm256_add_ps(_mm256_mul_ps(z, z), _mm256_mul_ps(zi, zi)), x);
zi = _mm256_add_ps(_mm256_mul_ps(_mm256_mul_ps(_mm256_set1_ps(2.0f), z), zi), y);
__m256 finished = _mm256_cmp_ps(_mm256_add_ps(_mm256_mul_ps(z, z), _mm256_mul_ps(zi, zi)), four, _CMP_LT_OS);
vals = _mm256_blendv_ps(_mm256_add_ps(vals, one), vals, finished);
if (_mm256_testc_ps(_mm256_cmp_ps(vals, vals, _CMP_EQ_UQ), finished) != 0)
break;
}
_mm256_store_ps(&results[ix + (iy * RESOLUTION)], vals);
}
}


High Priest Agripel:
Since the dark Magus himself asked you for it, here's the key.
You'd still better not use it yourself if you can help it.
Take the key


Final Fantasy Randomizer: HMS Jayne
Select a
Final Fantasy I & II: Dawn of Souls
ROM to randomize:
Upload ROM
Select options for the randomized ROM:
Flagstring:
Shuffle Key Items:
Encounter toggle:
Magic shuffle:
Treasure shuffle:
Formation shuffle:
Exp Scale:
Optionally, enter a seed value:
Seed:
Seeds may be up to 10 ASCII characters long.
Submit ROM:


Elmhurst College. Elmhurst, Illinois
Bachelor of Science: May 2016
Major: Computer Programming & Entertainment Technology
Minor: Computer Science
● Certified Scrum Master
● Full Stack – LAMP Stack
● C++, C#, SQL.
● Visual Studio.
● Microsoft Office.
● Agile Methodology
● Enterprise Software: SAP, Solomon, Sales Force.
● Immaculate communication, team synergy
● Unity - Collaborative group effort, made a first person shooter game in over a semester.
● Global Game Jam Programmer/Designer 2013-19. Design a video game from scratch with a team in 48 hours.
● Working knowledge of HTML code and Visual Basic code.
● Designed multi-page websites in HTML code.
● Collective group effort, designed and programmed multi-level, platformer game in 3D Unreal Engine.
● Designed a platformer in HTML5 Java script.
● Designed ActionScript 3.0 games.
● Experienced with Matlab.
● COBO International – Software Engineer
Design and programming of digital user interfaces on machinery with C/C++ code
Implementation of management and client specifications on new models of technology
June 2018 – November 2018
● Collegis – Technical Support Specialist
Troubleshoot with faculty and student on technical issues
Instruct new users on how to use new platforms
March 2018 – May 2018
● Ajilon, Contract Data Analyst – Rational USA
Partnered with the managers on technical projects
Prepare database for new module roll-out
September 2017 – December 2017
● Robert Half Technology, Contract Data Analyst – Securitas
Enhanced information retrieval of company’s data
Managed data migration between platforms
November 2016 – February 2017
● Achieved Eagle Scout - Coordinated Eagle Scout Project, food drive fed 500+ plus people
● Accomplished 8 years as a cross country and track athlete
● Coordinated campus-wide events, 100+ person turnout
● President, Video Game Developers Society, Elmhurst College
Aug 2012 – May 2016
● President, Public Relations, G.E.E.C. Elmhurst College
Aug 2012 – May 2016
● Alpha Phi Omega, Community Service Fraternity
Aug 2014 – May 2016
● Cross Country/Track and Field, Elmhurst College
Aug 2012 – May 2016
● Habitat for Humanity, Elmhurst College
Aug 2012 – May 2013


Shiboken
Shiboken is a fundamental piece on the Qt for Python project that serves two purposes:
Generator: Extract information from C or C++ headers and generate CPython code that allow to bring C or C++ projects to Python. This process uses a library called ApiExtractor which internally uses Clang.
Module: An utility Python module that exposed new Python types, functions to handle pointers, among other things, that is written in CPython and can use independently of the generator.


We invite you to participate in the Dialogue 2021 competition on Russian News Clustering and Headline Generation. The task consists of three tracks: clustering, headline selection, and headline generation. The goal of this competition is to collect and compare different approaches to text clustering and choosing the best headline for the resulting clusters.
News documents were taken from the Telegram Data Clustering Contest. Annotations were collected via Yandex.Toloka crowdsourcing platform.
The competition consists of three stages:
Discussion group in Telegram: https://t.me/joinchat/Irkkpio1xnDRowI_


Source code for util
# Copyright 2014 Modelling, Simulation and Design Lab (MSDL) at
# McGill University and the University of Antwerp (http://msdl.cs.mcgill.ca/)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Common utility functions used in PyPDEVS
"""
import pypdevs.middleware as middleware
from pypdevs.MPIRedirect import MPIRedirect
from collections import defaultdict
EPSILON = 1E-6
try:
import cPickle as pickle
except ImportError:
import pickle
[docs]def broadcastModel(data, proxies, allow_reinit, scheduler_locations):
"""
Broadcast the model to simulate to the provided proxies
:param data: data to be broadcasted to everywhere
:param proxies: iterable containing all proxies
:param allowReinit: should reinitialisation be allowed
"""
if (len(proxies) == 1) and not allow_reinit:
# Shortcut for local simulation with the garantee that no reinits will happen
proxies[0].sendModel(data, scheduler_locations[0])
return
# Otherwise, we always have to pickle
pickled_data = pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
if len(proxies) == 1:
proxies[0].saveAndProcessModel(pickled_data, scheduler_locations[0])
else:
for i, proxy in enumerate(proxies[1:]):
# Prepare by setting up the broadcast receiving
proxy.prepare(scheduler_locations[i+1])
# Pickle the data ourselves, to avoid an MPI error when this goes wrong (as we can likely back-up from this error)
# Broadcast the model to everywhere
middleware.COMM_WORLD.bcast(pickled_data, root=0)
# Immediately wait for a barrier, this will be OK as soon as all models have initted their model
# Still send to ourselves, as we don't receive it from the broadcast
# Local calls, so no real overhead
proxies[0].sendModel(data, scheduler_locations[0])
proxies[0].setPickledData(pickled_data)
middleware.COMM_WORLD.barrier()
[docs]def broadcastCancel():
"""
Cancel the broadcast receiving in a nice way, to prevent MPI errors
"""
middleware.COMM_WORLD.bcast(None, root=0)
[docs]def toStr(inp):
"""
Return a string representation of the input, enclosed with ' characters
:param inp: the input value
:returns: string -- input value, enclosed by ' characters
"""
return "'%s'" % inp
[docs]def addDict(destination, source):
"""
Adds 2 dicts together in the first dictionary
:param destination: the destination dictionary to merge the source into
:param source: the dictionary to merge in
.. note:: the *destination* parameter will be modified and no return value is provided. The *source* parameter is not modified.
"""
for i in source:
destination[i] = destination.get(i, 0) + source[i]
[docs]def allZeroDict(source):
"""
Checks whether or not a dictionary contains only 0 items
:param source: a dictionary to test
:returns: bool -- whether or not all entries in the dictionary are equal to zero
"""
for i in source.values():
if i != 0:
return False
return True
[docs]def runTraceAtController(server, uid, model, args):
"""
Run a trace command on our version that is running at the constroller
:param server: the server to ask the proxy from
:param uid: the UID of the tracer (identical throughout the simulation)
:param model: the model that transitions
:param args: the arguments for the trace function
"""
to_run = easyCommand("self.tracers.getByID(%i).trace" % uid,
args).replace("\n", "\\n")
if server.getName() == 0:
server.getProxy(0).delayedAction(model.time_last, model.model_id, to_run)
else:
server.queueMessage(model.time_last, model.model_id, to_run)
[docs]def easyCommand(function, args):
"""
Easy wrapper to create a string representation of function calls
:param function: the function should be called
:param args: list of all the arguments for the function
:returns: str -- string representation to be passed to *exec*
"""
text = str(function) + "("
for i in range(len(args)):
if i != 0:
text += ", "
text += str(args[i])
text += ")"
return text
[docs]class DEVSException(Exception):
"""
DEVS specific exceptions
"""
[docs] def __init__(self, message="not specified in source"):
"""
Constructor
:param message: error message to print
"""
Exception.__init__(self, message)
def __str__(self):
"""
String representation of the exception
"""
return "DEVS Exception: " + str(self.message)
[docs]class QuickStopException(Exception):
"""
An exception specifically to stop the simulation and perform a relocation ASAP
"""
[docs] def __init__(self):
Exception.__init__(self, "(none)")
def __str__(self):
"""
Should be unused
"""
return "Quick Stop Exception"
[docs]def saveLocations(filename, model_locations, model_ids):
"""
Save an allocation specified by the parameter.
:param filename: filename to save the allocation to
:param modellocations: allocation to save to file
:param model_ids: all model_ids to model mappings
"""
# Save the locations
f = open(filename, 'w')
for model_id in model_locations:
# Format:
# model_id location fullname
f.write("%s %s %s\n" % (model_id,
model_locations[model_id],
model_ids[model_id].getModelFullName()))
f.close()
[docs]def constructGraph(models):
"""
Construct a graph from the model, containing the weight (= number of messages) on a connection
between two components.
:param models: the root model to use for graph construction
:returns: dict -- all from-to edges with their number of events
"""
edges = defaultdict(lambda: defaultdict(int))
for model in models.component_set:
for outport in model.OPorts:
for inport in outport.outline:
edges[outport.host_DEVS][inport.host_DEVS] += outport.msg_count
return edges


public class TokenReader extends LineReader
do line = readline Convert.tokenizeString(line)Using this class takes less memory, because the line is read in tokens. For example, suppose a line contains 200 chars, and splitting them contains 10 tokens of 20 chars each. Using the first approach (readline/tokenizeString), the readline will create a string with 200 chars, then that totalcross.lang.String will be tokenized into 10 smaller strings of 20 chars each.
;a;;
is returned as {"","a","",""}
.
TokenReader reader = new TokenReader(new File("text.csv",File.READ_WRITE), ','); totalcross.lang.String[] tokens; while ((tokens = reader.readTokens()) != null) { ... do whatever you want with the tokens. }And here's another sample that parses from a string:
totalcross.lang.String lines = "a;;;a;\na;;;a; \nb;;b;b;b \nb;;b;b;b;\nb\nb;\n;b\n;\n b ;\n b \n b \n b \nb \n b"; totalcross.lang.String ll[] = Convert.tokenizeString(lines,'\n'); TokenReader tk = new TokenReader(new ByteArrayStream(lines.getBytes()),';'); tk.doTrim = true; totalcross.lang.String []line; for (int j =0; ((line = tk.readTokens()) != null); j++) { Vm.debug('"'+ll[j]+'"'); for (int i =0; i < line.length; i++) Vm.debug(i+": '"+line[i]+"'"); Vm.debug(""); }The output is:
"a;;;a;" 0: 'a' 1: '' 2: '' 3: 'a' 4: '' "a;;;a; " 0: 'a' 1: '' 2: '' 3: 'a' 4: '' "b;;b;b;b " 0: 'b' 1: '' 2: 'b' 3: 'b' 4: 'b' "b;;b;b;b;" 0: 'b' 1: '' 2: 'b' 3: 'b' 4: 'b' 5: '' "b" 0: 'b' "b;" 0: 'b' 1: '' ";b" 0: '' 1: 'b' ";" 0: '' 1: '' " b ;" 0: 'b' 1: '' " b " 0: 'b' " b " 0: 'b' " b " 0: 'b' "b " 0: 'b' " b" 0: 'b'Note that this class already uses a buffer for faster detection of the newline and delimiters. Don't use TokenReader with a BufferedStream, it's nonsense and it will throw a warning on the desktop.
| Modifier and Type | Field and Description |
|---|---|
protected char |
delimiter |
doTrim, f, maxTries, ofs, readBuf, returnEmptyLines
| Constructor and Description |
|---|
TokenReader(Stream f,
char delimiter)
Constructs a new TokenReader and sets maxTries accordingly to the type of
class: 10 if its a Socket or a PortConnector; 0, otherwise.
|
TokenReader(Stream f,
char delimiter,
byte[] buffer,
int start,
int len)
Constructs a new TokenReader and sets maxTries accordingly to the type of
class: 10 if its a Socket or a PortConnector, 0 otherwise.
|
| Modifier and Type | Method and Description |
|---|---|
String |
readLine()
Cannot be used; throws a RuntimeException.
|
String[] |
readTokens()
Returns the next tokens available in this stream or null if none.
|
getStream, readMore, reuse, setStream
public TokenReader(Stream f, char delimiter) throws IOException
IOException
public TokenReader(Stream f, char delimiter, byte[] buffer, int start, int len) throws IOException
IOException
public String readLine()
readLine
in class LineReader
readTokens()
public String[] readTokens() throws IOException
IOException


Changes the URL of the remote repository.
git remote set-url origin <url>
to change the URL of the remote repository to <url>
.git remote set-url origin <url>
git remote set-url origin https://github.com/30-seconds/30-seconds-of-code
# The remote URL is now "https://github.com/30-seconds/30-seconds-of-code"


Technology Putting users at the heart of everything we do
The scale of the digital technology challenge in DWP is one of the largest in the UK today. Our IT systems underpin the annual processing of 5m benefit claims, payment of £167.6bn in benefits and pensions, and support over 90,000 users across 850 locations.
Digital technology in DWP is transforming to become a leading example of a nimble, digital enterprise, with deep experts who are motivated by the challenge of working on large-scale applications running on massive platforms.
We will design based on user needs, for a future where new and current systems will co-exist. Delivery teams will be accountable for test-driven development, business analysis, testing, devops, data modelling, repositories, analytics and visualisation, working closely with a cyber-security team.
We will reuse and repurpose technologies to accelerate delivery, building reusable platform utilities and enablers as well as reusing a library of services, objects, assets, rules and roles including roll-forward of architectures.
This represents a major shift from a traditional outsourcing model, making this one of the most exciting times to join a major government department that delivers services that support the lives of 22m citizens.
Our digital technologists will build reusable Code Components and Service Libraries using UX (Angular JS, D3, Bootstrap: HTML5 and CSS4), ExpressJS, Mashery API Registry, RabbitMQ, JSON, MVC, Micro-service design patterns and RESTful interfaces.
To deliver this large-scale transformation we need to complement our existing expertise with digital technology skills with a focus on Javascript development/Java as an exception, REDIS, MongoDB, Mongoose and Cassandra DB, Devops (Chef, Statsd, Docker and Flyway) and BigData and Visualisation (Hadoop, R, Graph, D3 and Tableau).
We have 50m lines of code and support over 1,000 applications
{% include "includes/call_to_action.html" %}


Kickstarter is a public benefit corporation for crowd-funding.
The Goal of Kickstarter Statistic is to compare
projects from every country it operates in.
Kickstarter Statistics is a project of
Benedikt Strasser, Matthias Hermann and Marcel Mück
from the University of Regensburg
The map and "Launched At Hours" bar chart are based on the datasets from March 2016 - February 2018 provided by webrobots.
Other charts are using official kickstarter statistics from 21.07.2018.
Each box represents a category. The size of the box is proportional to it`s value. The data which has been used is from the official kickstarter site.
The spider chart displays the success rate of projects in the given category. The data which has been used is from the official kickstarter site.
The bar chart displays how many projects have been launched at which hour in German time. The data which has been used is provided by webrobots.


Submit Samples
View Submissions
Logout
Select AAP domain
Select checklist for validation
Search
Recent Submissions
Please log in again to see all immediate submissions.
Submission ID
Submission Status
Submission Date
Download Link
Download


{{description if description != None}}
{{operation.summary if operation.summary != None}}
Implementation notes: {{operation.notes}}
Response Class: {{operation.responseClass}}


Style 1
Style 2
Style 3
Style 4
Style 5
Style 6
Style 7
Style 8
Style 9
Style 10
Style 11
Style 12
Style 13
Style 14
Style 15
Style 16
Style 17
Style 18
Style 19
Style 20
Style 21
Style 22
Style 23
Style 24
Style 25
Style 26
Style 27
Style 28
Style 29
Style 30
$2.99 /month
200Mb Monthly traffic
10Gb disk space
5 Domains
5 Subdomains
Unlimited email support
PHP 5 Support
MySql support
$6.99 /month
400Mb Monthly traffic
20Gb disk space
10 Domains
10 Subdomains
Unlimited email support
PHP 5 Support
MySql support
$9.99 /month
700Mb Monthly traffic
50Gb disk space
15 Domains
20 Subdomains
Unlimited email support
PHP 5 Support
MySql support
$19.99 /month
3500Mb Monthly traffic
150Gb disk space
30 Domains
27 Subdomains
Unlimited email support
PHP 5 Support
MySql support


We examine methodologies!
- grow dynamic niches
- enable clicks-and-mortar metrics
- e-enable sticky platforms
- grow visionary systems
- reinvent virtual e-services
- brand frictionless mindshare
- revolutionize sticky mindshare
- productize granular initiatives
- recontextualize 24/7 vortals
- synthesize virtual architectures
- seize cross-media web-readiness
- strategize mission-critical technologies
- embrace bleeding-edge partnerships
- streamline front-end systems
- scale wireless infomediaries


Business Advisory Practice
"Purposeful Business"
We help you transform both your strategy and execution.
As an extension of your senior team, we recognize the unique strengths and challenges of your business, and we raise performance to create value for your shareholders and opportunity for your employees by focusing on:
We insure you deliver superior top and bottom line results. Our agile approach is built to begin driving results fast.
What are your needs? Are you looking for:
We build your acumen so you will implement both transformational strategy and execution.
The approach to every business needs to be different.
Unlike most methodologies, we help you apply both strategy and execution tactics to your company in the way that has the most impact and allows you to move fast to drive significant top and bottom line results.
We help you define a Transformational Strategy for your business centered around three simple elements:
We build a Transformational High Execution Business Model by working with you to move through a clear progression of 6 tactics:
We improve the Strategy and Execution of your business!
Let us help you grow your acumen for these two key drivers...
so you can transform and grow sales and profits!
"We make sure we understand your company’s purpose and work purposefully towards that end."
Steven Singer, CEO and Owner


| DOSE: |
10 leaves |
buccal |
Coleus |
(leaves) |
My wife was reading some information about Salvia Divinorum, which we are desperate to try, but has recently (last year) been pre-emptivly banned by the Aussie Goverment, and so came across an article about Coleus Blumei being the another alternative for Mexican Indians as a divinitive aid. She pointed out that we have one growing in the garden!
I do not do much stuff, I do not drink (very little indeed) or smoke, and have not had anything of stimulants for a good 3 months or so (hence looking for natural highs). Well, after some research about toxitity, found nothing concrete on it, i tried a little experiment. I rolled 10 leaves into a quid, and put it in my cheek. I did not swallow at all, but just let the spit build up over 20 mins, then spat it out.  Then sat watching TV in the lounge. Time around 8.30pm.
Effects
-------
Felt a little wierd after 20 mins or so, a bit drifty, and was watching TV, when my perception of what I was looking at changed dramatically... Robert Dinero's head was suddenly square, and the next character on telly had 50% of his head a complete right angle.. only lasted a sec or so, as I did a double take, but It was a definate distortion of my vision. Did not happen again for the rest of the night....
Rest of the night went without anything to mention apart from feeling a little bit strange and mildly stimulated, then went to bed as normal 3 hours later but i had a TERRIBLE night's sleep. Was very aware that I was semi-consious all night, vaguely aware that I was asleep.... Not unpleasant, but the following day was quite a depressed feeling, which is not like me at all....
Will try this again at some point, but with a much higher dose. One thing however, is that I found some information that is relevant to
health warnings.
Experience Reports are the writings and opinions of the individual authors who submit them.
Some of the activities described are dangerous and/or illegal and none are recommended by Erowid.


Incorrect. Please try again.