diff --git a/.gitignore b/.gitignore index 07b1e6cf9..eea0abbf2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ _build .DS_Store rel/vars/*_vars.config riak.png +log/* +rebar3.crashdump +.eunit diff --git a/Makefile b/Makefile index e8b70dc33..02585f0e1 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ BASE_DIR = $(shell pwd) ERLANG_BIN = $(shell dirname $(shell which erl 2>/dev/null) 2>/dev/null) REBAR ?= $(BASE_DIR)/rebar3 OVERLAY_VARS ?= -TEST_IGNORE ?= riak +TEST_IGNORE ?= lager riak basho_bench TEST_DEPS_DIR ?= _build/test/lib REL_DIR ?= _build/default/rel DEPS = $(patsubst $(TEST_DEPS_DIR)/%, %, $(wildcard $(TEST_DEPS_DIR)/*)) @@ -28,8 +28,8 @@ compile: $(REBAR) compile deps: - $(if $(HEAD_REVISION),$(warning "Warning: you have checked out a tag ($(HEAD_REVISION)) and should use the locked-deps target")) - $(REBAR) get-deps + $(if $(HEAD_REVISION),$(warning "Warning: you have checked out a tag ($(HEAD_REVISION)) and should use the compile target")) + $(REBAR) upgrade clean: testclean $(REBAR) clean @@ -37,9 +37,6 @@ clean: testclean distclean: clean devclean relclean ballclean @rm -rf _build -locked-deps: - $(REBAR) upgrade - ## ## Test targets ## @@ -65,24 +62,27 @@ test-deps : compile testclean $(patsubst %, testdep-%, $(TEST_DEPS)) echo Tested the dependencies: $(TEST_DEPS) # Test each dependency individually in its own VM -test : test-deps +test : testclean eunit test-deps @if test -s $(TEST_LOG_FILE) ; then \ - cat $(TEST_LOG_FILE) | grep FAILED && \ - exit `cat $(TEST_LOG_FILE) | grep FAILED | wc -l`; \ - fi - $(REBAR) eunit - - + cat $(TEST_LOG_FILE) && \ + exit `cat $(TEST_LOG_FILE) | grep FAILED | wc -l`; \ + fi ## ## Release targets ## -rel: locked-deps compile +rel: compile $(REBAR) as rel release + cp -a _build/rel/rel/riak rel/ -rel-rpm: locked-deps compile - $(REBAR) as rel,rpm release +rel-rpm: compile + $(REBAR) as rpm release + cp -a _build/rpm/rel/riak rel/ + +rel-deb: compile + $(REBAR) as deb release + cp -a _build/deb/rel/riak rel/ relclean: rm -rf $(REL_DIR) @@ -110,7 +110,7 @@ $(eval devrel : $(foreach n,$(SEQ),dev$(n))) dev% : all rel/gen_dev dev$* rel/vars/dev_vars.config.src rel/vars/$*_vars.config - $(REBAR) release -o dev/dev$* --overlay_vars rel/vars/$*_vars.config + $(REBAR) as dev release -o dev/dev$* --overlay_vars rel/vars/$*_vars.config stagedev% : all rel/gen_dev dev$* rel/vars/dev_vars.config.src rel/vars/$*_vars.config @@ -212,46 +212,6 @@ REVISION = $(shell echo $(REPO_TAG) | sed -e 's/^$(REPO)-//') # Changes to 1.0.3 or 1.1.0pre1 from example above MAJOR_VERSION ?= $(shell echo $(REVISION) | sed -e 's/\([0-9.]*\)-.*/\1/') - -## -## Release tarball creation -## Generates a tarball that includes all the deps sources so no checkouts are necessary -## - -# Use git archive make a clean copy of a repository at a current -# revision and copy to a new directory -archive_git = git archive --format=tar --prefix=$(1)/ HEAD | (cd $(2) && tar xf -) - -# Alternative to git archive to remove .git directory, but not any -# other files outside of the source tree (used for eleveldb which -# brings in leveldb) -clean_git = cp -R ../../$(1) $(2)/deps/ && find $(2)/$(1) -name .git -type d | xargs rm -rf - -# Determines which function to call. eleveldb is treated as a special case -archive = if [ "$(1)" = "deps/eleveldb" ]; then \ - $(call clean_git,$(1),$(2)); \ - else \ - $(call archive_git,$(1),$(2)); \ - fi - - -# Checkout tag, fetch deps (so we don't have to do it multiple times) and collect -# the version of all the dependencies into the MANIFEST_FILE -CLONEDIR ?= riak-clone -MANIFEST_FILE ?= dependency_manifest.git -get_dist_deps = mkdir distdir && \ - git clone . distdir/$(CLONEDIR) && \ - cd distdir/$(CLONEDIR) && \ - git checkout $(REPO_TAG) && \ - $(MAKE) locked-deps && \ - echo "- Dependencies and their tags at build time of $(REPO) at $(REPO_TAG)" > $(MANIFEST_FILE) && \ - for dep in deps/*; do \ - cd $${dep} && \ - printf "$${dep} version `git describe --long --tags 2>/dev/null || git rev-parse HEAD`\n" >> ../../$(MANIFEST_FILE) && \ - cd ../..; done && \ - LC_ALL=POSIX && export LC_ALL && sort $(MANIFEST_FILE) > $(MANIFEST_FILE).tmp && mv $(MANIFEST_FILE).tmp $(MANIFEST_FILE); - - # Name resulting directory & tar file based on current status of the git tag # If it is a tagged release (PKG_VERSION == MAJOR_VERSION), use the toplevel # tag as the package name, otherwise generate a unique hash of all the @@ -261,41 +221,6 @@ get_dist_deps = mkdir distdir && \ NAME_HASH = $(shell git hash-object distdir/$(CLONEDIR)/$(MANIFEST_FILE) 2>/dev/null | cut -c 1-8) PKG_ID := $(REPO_TAG) -# To ensure a clean build, copy the CLONEDIR at a specific tag to a new directory -# which will be the basis of the src tar file (and packages) -# The vsn.git file is required by rebar to be able to build from the resulting -# tar file -build_clean_dir = cd distdir/$(CLONEDIR) && \ - $(call archive_git,$(PKG_ID),..) && \ - cp $(MANIFEST_FILE) ../$(PKG_ID)/ && \ - mkdir ../$(PKG_ID)/deps && \ - for dep in deps/*; do \ - cd $${dep} && \ - $(call archive,$${dep},../../../$(PKG_ID)) && \ - mkdir -p ../../../$(PKG_ID)/$${dep}/priv && \ - printf "`git describe --long --tags 2>/dev/null || git rev-parse HEAD`" > ../../../$(PKG_ID)/$${dep}/priv/vsn.git && \ - cd ../..; \ - done - - -distdir/$(CLONEDIR)/$(MANIFEST_FILE): - $(if $(REPO_TAG), $(call get_dist_deps), $(error "You can't generate a release tarball from a non-tagged revision. Run 'git checkout ', then 'make dist'")) - -distdir/$(PKG_ID): distdir/$(CLONEDIR)/$(MANIFEST_FILE) - $(call build_clean_dir) - -distdir/$(PKG_ID).tar.gz: distdir/$(PKG_ID) - tar -C distdir -czf distdir/$(PKG_ID).tar.gz $(PKG_ID) - -dist: distdir/$(PKG_ID).tar.gz - cp distdir/$(PKG_ID).tar.gz . - -ballclean: - rm -rf $(PKG_ID).tar.gz distdir - -pkgclean: ballclean - rm -rf package - ## ## Packaging targets ## @@ -305,8 +230,9 @@ pkgclean: ballclean PKG_VERSION = $(shell echo $(PKG_ID) | sed -e 's/^$(REPO)-//') package: - git archive --format=tar HEAD | gzip >rel/pkg/out/riak-$(PKG_ID).tar.gz - $(MAKE) -C rel/pkg/ -f Makefile + mkdir rel/pkg/out/riak-$(PKG_ID) + git archive --format=tar HEAD | gzip >rel/pkg/out/$(PKG_ID).tar.gz + $(MAKE) -f rel/pkg/Makefile packageclean: rm -rf rel/pkg/out/* @@ -316,10 +242,10 @@ packageclean: export PKG_VERSION PKG_ID PKG_BUILD BASE_DIR ERLANG_BIN REBAR OVERLAY_VARS RELEASE # Package up a devrel to save time later rebuilding it -pkg-devrel: locked-deps devrel +pkg-devrel: devrel echo -n $(PKG_REVISION) > VERSION tar -czf $(PKG_ID)-devrel.tar.gz dev/ VERSION rm -rf VERSION -pkg-rel: locked-deps rel +pkg-rel: rel tar -czf $(PKG_ID)-rel.tar.gz -C rel/ . diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 676944f2f..9b5c52493 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,144 @@ +# Riak KV 3.0 Release Notes + +This major release allows Riak to run on OTP versions 20, 21 and 22 - but is not fully backwards-compatible with previous releases. Some limitations and key changes should be noted: + +- It is not possible to run this release on any OTP version prior to OTP 20. Testing of node-by-node upgrades is the responsibility of Riak customers, there has been no comprehensive testing of this upgrade managed centrally. Most customer testing of upgrades has been spent on testing an uplift from 2.2.x and OTP R16 to 3.0 and OTP 20, so this is likely to be the safest transition. + +- This release will not by default include Yokozuna support, but Yokozuna can be added in by reverting the commented lines in rebar.config. There are a number of riak_test failures with Yokozuna, and these have not been resolved prior to release. More information on broken tests can be found [here](https://github.com/basho/yokozuna/pull/767). Upgrading with yokozuna will be a breaking change, and data my be lost due to the uplift in solr version. Any migration will require bespoke management of any data within yokozuna. + +- Packaging support is not currently proven for any platform other than CentOS, Debian or Ubuntu. Riak will build from source on other platforms - e.g. `make locked-deps; make rel`. + +- As part of the release there has been a comprehensive review of all tests across the dependencies (riak_test, eunit, eqc and pulse), as well as removal of all dialyzer and xref warnings and addition where possible of travis tests. The intention is to continue to raise the bar on test stability before accepting Pull Requests going forward. + +- If using riak_client directly (e.g. `{ok, C} = riak:local_client()`), then please use `riak_client:F(*Args, C)` not `C:F(*Args)` when calling functions within riak_client - the latter mechanism now has issues within OTP 20+. + +- Instead of `riak-admin` `riak admin` should now be used for admin CLI commands. + +Other than the limitations listed above, the release should be functionally identical to Riak KV 2.9.7. Throughput improvements may be seen as a result of the OTP 20 upgrade on some CPU-bound workloads. For disk-bound workloads, additional benefit may be achieved by upgrading further to OTP 22. + +# Riak KV 2.9.7 Release Notes + +This release improves the stability of Riak when running with Tictac AAE in parallel mode: + +- The aae_exchange schedule will back-off when exchanges begin to timeout due to pressure in the system. + +- The aae_runner now has a size-limited queue of snapshots for handling exchange fetch_clock queries. + +- The aae tree rebuilds now take a snapshot at the point the rebuild is de-queued for work, not at the point the rebuild is added to the queue. + +- The loading process will yield when applying the backlog of changes to allow for other messages to interleave (that may otherwise timeout). + +- The aae sub-system will listen to back-pressure signals from the aae_keystore, and ripple a response to slow-down upstream services (and ultimately the riak_kv_vnode). + +- It is possible to accelerate and decelerate AAE repairs by setting riak_kv application variables during running (e.g `tictacaae_exchangetick`, `tictacaae_maxresults`), and also log AAE-prompted repairs using `log_readrepair`. + +The system is now stable under specific load tests designed to trigger AAE failure. However, parallel mode should still not be used in production systems unless it has been subject to environment-specific load testing. + +# Riak KV 2.9.6 Release Notes + +Withdrawn. + +# Riak KV 2.9.5 Release Notes + +Withdrawn. + +# Riak KV 2.9.4 Release Notes + +This release replaces the Riak KV 2.9.3 release, extending the issue resolution in kv_index_tictactree to detect other files where file truncation means the CRC is not present. + +This release has a key [outstanding issue](https://github.com/basho/riak_kv/issues/1765) when Tictac AAE is used in parallel mode. On larger clusters, this has been seen to cause significant issues, and so this feature should not be used other than in native mode. + +# Riak KV 2.9.3 Release Notes - NOT TO BE RELEASED + +This release is focused on fixing a number of non-critical issues: + +- [An issue with Tictac AAE](https://github.com/basho/riak_kv/issues/1759) failing to clean-up state correctly on node leaves, which may lead to false repairs on re-joining a node (until tree rebuilds occur). + +- [An issue with using the leveled backend in Riak CS](https://github.com/basho/riak_kv/issues/1758) releated to the ordered object fold not respecting the passed in range and crashing on hitting an out of range value. + +- [An issue with duplicate PUTs](https://github.com/basho/riak_kv/issues/1754) related to retries of PUT coordinater forward requests due to lost ACKs. + +- [An issue with Tictac AAE startup](https://github.com/martinsumner/kv_index_tictactree/issues/74) caused by a failure of the corrupt file detection to handle a situation where file truncation means the CRC is not present. + + +# Riak KV 2.9.2 Release Notes + +This release includes: + +- An extension to the `node_confirms` feature so that `node_confirms` can be [tracked on GETs as well as PUTs](https://github.com/basho/riak_kv/issues/1750). This is provided so that if an attempt to PUT with a `node_confirms` value failed, a read can be made which will only succeed if repsonses from sufficient nodes are received. This does not confirm absolutely that the actual returned response is on sufficient nodes, but does confirm nodes are now up so that anti-entropy mechanisms will soon resolve any missing data. + +- Support for building [leveldb on 32bit platforms](https://github.com/basho/leveldb/commit/14c57fe018402b3271cc8ac070fbd620b6ff7798). + +- Improvements to reduce the cost of journal compaction in leveled when there are large numbers of files containing mainly skeleton key-changes objects. The cost of scoring all of these files could have a notable impact on read loads when spinning HDDs are used (although this could be mitigated by running the journal compaction less frequently, or out of hours). Now an attempt is made to reduce this scoring cost by reading the keys to be scored in order, and scoring keys relatively close together. This will reduce the size of the disk head movements required to complete the scoring process. + +- The abilty to switch the configuration of leveled [journal compaction to using `recalc` mode](https://github.com/martinsumner/leveled/issues/306), and hence avoid using skeleton key changes objects altogether. The default remains `retain` mode, the switch from `retain` mode to enabling `recalc` is supported without any data modification (just a restart required). There is though, no safe way other than leaving the node from the cluster (and rejoining) to revert from `recalc` back to `retain`. The use of the `recalc` strategy [can be enabled via configuration](https://github.com/basho/riak_kv/blob/riak_kv-2.9.2/priv/riak_kv.schema#L1120-L1138). The use of `recalc` mode has outperformed `retain` in tests, when both running journal compaction jobs, and recovering empty ledgers via journal reloads. + +- An improvement to the efficiency of [compaction in the leveled LSM-tree based ledger](https://github.com/martinsumner/leveled/issues/311) with large numbers of tombstones (or modified index entries), by using a `grooming` selection strategy 50% of the time when selecting files to merge rather than selecting files at random each time. The `grooming` selection, will take a sample of files and merge the one with the most tombstones. The use of the grooming strategy is not configurable, and will have no impact until the vast majority of SST files have been re-written under this release. + +# Riak KV 2.9.1 Release Notes + +This release adds a number of features built on top of the Tictac AAE feature made available in 2.9.0. The new features depend on Tictac AAE being enabled, but are backend independent. The primary features of the release are: + +- A new combined full-sync and real-time replication system `nextgenrepl`, that is much faster and more efficient at reconciling overall state of clusters (e.g. full-sync). This feature is explained further [here](https://github.com/basho/riak_kv/blob/riak_kv-2.9.1.1/docs/NextGenREPL.md). + +- A mechanism for requesting mass deletion of objects on expiry, and mass reaping of tombstones after a time to live. This is not yet an automated, scheduled, set of garbage collection processes, it is required to be triggered by an operational process. This feature is explained further [here](https://github.com/basho/riak_kv/blob/riak_kv-2.9.1.1/docs/ReapErase.md). + +- A safe method of listing buckets regardless of backend chosen. Listing buckets had previously not been production safe, but can still be required in production environments - it can now be managed safely via an aae_fold. For more information on using the HTTP API for all aae_folds see [here](https://github.com/basho/riak_kv/blob/develop-2.9/src/riak_kv_wm_aaefold.erl#L19-L91). Or to use via the riak erlang client see [here](https://github.com/basho/riak-erlang-client/blob/riak_kv-2.9.1/src/riakc_pb_socket.erl#L1344-L1932). + +- A version uplift of the internal `ibrowse` client, a minor `riak_dt` fix to resolve issues of unit test reliability, a fix to help build (the now deprecated) erlang_js in some environments, and the removal of `hamcrest` as a dependency. + +Detail of volume testing related to the replication uplift can be found [here](https://github.com/martinsumner/riak_testing_notes/blob/master/Release%202.9.1%20-%20Volume%20Tests.md). + +# Riak KV 2.9.0 Release Notes - Patch 5 + +This patch release is primarily required to improve two scenarios where leveled's handling of file-system corruption was unsatisfactory. One scenario related to [corrupted blocks that fail CRC checks](https://github.com/martinsumner/leveled/issues/298), another related to failing to consistently check that [a file write had been flushed to disk](https://github.com/martinsumner/leveled/issues/301). + +The patch also removes `eper` as a dependency, instead having direct dependencies on `recon` and `redbug`. This as a side effect, resolves the background noise of CI failures related to eper eunit test inconsistency. + +The patch resurrects Fred Dushin's work to [improve the isolation between Yokozuna and Riak KV](https://github.com/basho/riak_kv/pull/1721). This, in the future will make it easier for those not wishing to carry Yokozuna as a dependency to deploy Riak without it. Any Yokozuna users should take extra care in pre-production testing of this patch. Testing time for Yokozuna within the Riak release cycle is currently very limited. + +The patch changes the handling by bitcask and eleveldb backends when file-systems are switched [into a read-only state](https://github.com/basho/riak_kv/issues/1714). This previously might lead to zombie nodes, nodes that are unusable never actually close due to failures, and so continue to participate in the ring. + +The patch changes the capability request for `HEAD` to be bucket-dependent, so that [HEAD requests are supported correctly in a multi-backend configuration](https://github.com/basho/riak_kv/issues/1719). + +Finally, the aae_fold features of 2.9.0 were previously only available via the HTTP API. Such folds can now also be [requested via the PB API](https://github.com/basho/riak_kv/pull/1732). + +# Riak KV 2.9.0 Release Notes - Patch 4 + +There are a number of fixes in this patch: + +- Mitigation, and further monitoring, to assist in [an issue](https://github.com/basho/riak_kv/issues/1707) where a riak_kv_vnode may attempt to PUT an object with empty contents. This will now check for this case regardless of the allow_mult setting, and prompt a PUT failure without crashing the vnode. + +- Tictac AAE will now, as with standard AAE, exchange between primary vnodes only for a partition (and not backfill fallbacks). This change [can be reversed by configuration](https://github.com/basho/riak_kv/blob/riak_kv-2.9.0p5/priv/riak_kv.schema#L102-L109), and this also helps with an issue with [delayed handoff due to Tictac AAE](https://github.com/basho/riak_kv/issues/1706). + +- A [change](https://github.com/basho/riak_kv/pull/1712) to allow the management via `riak attach` of the tokens used to ensure that the AAE database is not flooded due to relative under-performance of this store when compared to the vnode store. This allows for admin activities (like coordinated rebuilds) to be managed free of lock issues related to token depletion. This is partial mitigation for a broader [issue](https://github.com/basho/riak/issues/981), and the change also includes improved monitoring of the time it takes to clear trees during the rebuild process to help further investigate the underlying cause of these issues. + +- A [refactoring of the code](https://github.com/basho/riak_kv/pull/1710) used when managing a GET request with HEAD and GET responses interspersed. This is a refactor to improve readability, dialyzer support and also enhance the predictability of behaviour under certain race conditions. There is no known issue resolved by this change. + +- An uplift to the leveled version to add in an extra log (on cache effectiveness) and resolve an intermittent test failure. + +It is recommended that any 2.9.0 installations be upgraded to include this path, although if Tictac AAE is not used there is no immediate urgency to making the change. + +# Riak KV 2.9.0 Release Notes - Patch 3 + +An [issue](https://github.com/martinsumner/leveled/issues/287) was discovered in leveled, whereby following a restart of Riak and a workload of fetch requests, the backend demanded excess amounts of binary heap references. Underlying was an issue with the use of sub-binary references during the lazy load of slot header information after a SST file process restart. This has been resolved, and with [greater control added](https://github.com/martinsumner/leveled/blob/0.9.18/priv/leveled.schema#L86-L93) to force the ledger contents into the page cache at startup. + +A further [issue](https://github.com/martinsumner/leveled/issues/289) was discovered in long-running pre-production tests whereby leveled journal compaction could enter into a loop where it would perform compaction work, that failed to release space. This has been resolved, and some further safety checks added to ensure that memory usage does not grow excessively during the comapction process. As part of this change, an additional [configurable limit](https://github.com/martinsumner/leveled/blob/0.9.18/priv/leveled.schema#L75-L84) has been added on the number of objects in a leveled journal (CDB) file - now a file will be considered full when it hits either the space limit (previous behaviour) or the object limit. + +The issues resolved in this patch impact only the use of leveled backend, either directly or via the use of Tictac AAE. + +# Riak KV 2.9.0 Release Notes - Patch 2 + +An [issue](https://github.com/martinsumner/leveled/issues/285) with leveled holding references to binaries what could cause severe memory depletion, when a consecutive series of very large objects are received by a vnode. + +# Riak KV 2.9.0 Release Notes - Patch 1 + +An [issue](https://github.com/basho/riak_kv/issues/1699) was discovered whereby leveled would leak file descriptors under heavy write pressure (e.g. handoffs). + +# Riak KV 2.9.0 Release Notes + +See [here for notes on 2.9.0](doc/Release%202.9%20Series%20-%20Overview.md) + # Riak KV 2.2.5 Release Notes > This release is dedicated to the memory of Andy Gross. Thank you and RIP. @@ -33,7 +174,7 @@ and [fixes to tests](#developer-improvements) and the build/development process. [Log of Changes](#change-log-for-this-release) -[Previous Release Notes](releasenotes/RELEASE-NOTES-2_2_4.md) +[Previous Release Notes](RELEASE-NOTES-2_2_4.md) ## Improvements @@ -170,4 +311,4 @@ More details about the issue can be found in riak\_repl/782: [2.2.5 - \[enoent\] ## Previous Release NOTES -[Full history of Riak 2.0+ release notes](releasenotes/RELEASE-NOTES-2_2_4.md) +[Full history of Riak 2.0+ release notes](RELEASE-NOTES-2_2_4.md) diff --git a/apps/riak/src/riak.app.src b/apps/riak/src/riak.app.src index 23ae71d48..4b049872c 100644 --- a/apps/riak/src/riak.app.src +++ b/apps/riak/src/riak.app.src @@ -7,6 +7,7 @@ kernel, stdlib, lager, + lager_syslog, sasl, public_key, ssl, diff --git a/apps/riak/test/riak_schema_test.erl b/apps/riak/test/riak_schema_test.erl index 1efeea965..6c671b93e 100644 --- a/apps/riak/test/riak_schema_test.erl +++ b/apps/riak/test/riak_schema_test.erl @@ -1,6 +1,6 @@ -module(riak_schema_test). -include_lib("eunit/include/eunit.hrl"). --compile(export_all). +-compile([export_all, nowarn_export_all]). -define(riak_schema, "priv/riak.schema"). -define(eleveldb_schema, "_build/test/lib/eleveldb/priv/eleveldb.schema"). diff --git a/dialyzer.ignore-warnings b/dialyzer.ignore-warnings deleted file mode 100644 index bb5bf1838..000000000 --- a/dialyzer.ignore-warnings +++ /dev/null @@ -1,79 +0,0 @@ -### -### This is not a comment, just a pattern that is -### very unlikely to match anything that we care about. -### Don't put blank lines in this file. -### -### Items that can be ignored because of EUnit macros doing odd -### things or Dialyzer not finding erlang:load_nif/2 (which definitely exists). -### -Call to missing or unexported function erlang:load_nif/2 -The variable _ can never match since previous clauses completely covered the type 'false' -Contract for function that does not exist: riak_client:for_dialyzer_only_ignore/2 -riak_core_ring_manager.erl:324: Call to missing or unexported function riak_core_test_util:setup_mockring1/0 -riak_kv_delete.erl:107: The pattern 'true' can never match the type 'false' -### -### Items in third-party code that are harmless enough -### -js_mochijson2.erl:530: The variable __V can never match since previous clauses completely covered the type 'true' -mochijson2.erl:530: The variable __V can never match since previous clauses completely covered the type 'true' -mochiweb_request.erl:246: The variable Length can never match since previous clauses completely covered the type 'chunked' | 'undefined' | integer() | {'unknown_transfer_encoding',_} -mochiweb_request.erl:271: The call mochiweb:new_response({{_,port() | {'ssl',{'sslsocket',_,_}},_,_,_,_},binary() | maybe_improper_list() | integer(),gb_tree()}) contains an opaque term as 1st argument when a structured term of type {_,_,[any()] | tuple()} is expected -mochiweb_socket_server.erl:128: The call erlang:integer_to_list(S::maybe_improper_list()) will never return since it differs in the 1st argument from the success typing arguments: (integer()) -mochiweb_sup.erl:26: The pattern [Spec | _] can never match the type [] -watchdog.erl:267: The call lists:keyreplace(Key::{'log' | 'pid' | 'tcp' | 'udp' | integer(),_},1,Subs::maybe_improper_list(),Sub::fun((_) -> any())) will never return since it differs in the 4th argument from the success typing arguments: (any(),pos_integer(),maybe_improper_list(),tuple()) -### -### Warnings from auto-generated protobufs code. Hard/not-worth-it-now to fix. -leexinc.hrl:49: The pattern can never match the type <_,_,'error' | 'skip_token',[{atom(),_} | {'float',_,float()} | {'integer',_,_} | {'string',_,[any()]} | {'var',_,atom()}]> -leexinc.hrl:52: The pattern can never match the type <_,_,'error' | 'skip_token',[{atom(),_} | {'float',_,float()} | {'integer',_,_} | {'string',_,[any()]} | {'var',_,atom()}]> -leexinc.hrl:54: The pattern can never match the type <_,_,'error' | 'skip_token',[{atom(),_} | {'float',_,float()} | {'integer',_,_} | {'string',_,[any()]} | {'var',_,atom()}]> -leexinc.hrl:59: The pattern can never match the type <_,_,'error',[{atom(),_} | {'float',_,float()} | {'integer',_,_} | {'string',_,[any()]} | {'var',_,atom()}]> -leexinc.hrl:62: The pattern <_Rest, Line, {'error', S}, _Ts> can never match the type <_,_,'error',[{atom(),_} | {'float',_,float()} | {'integer',_,_} | {'string',_,[any()]} | {'var',_,atom()}]> -leexinc.hrl:118: The pattern can never match the type <_,_,'error' | 'skip_token'> -leexinc.hrl:121: The pattern can never match the type <_,_,'error' | 'skip_token'> -leexinc.hrl:123: The pattern can never match the type <_,_,'error' | 'skip_token'> -leexinc.hrl:128: The pattern can never match the type <_,_,'error'> -leexinc.hrl:131: The pattern can never match the type <_,_,'error'> -leexinc.hrl:192: The pattern can never match the type <_,_,'error' | 'skip_token',_> -leexinc.hrl:195: The pattern can never match the type <_,_,'error' | 'skip_token',_> -leexinc.hrl:197: The pattern can never match the type <_,_,'error' | 'skip_token',_> -leexinc.hrl:202: The pattern can never match the type <_,_,'error',_> -leexinc.hrl:205: The pattern can never match the type <_,_,'error',_> -leexinc.hrl:243: The pattern can never match the type <_,_,'error' | 'skip_token',_> -leexinc.hrl:246: The pattern can never match the type <_,_,'error' | 'skip_token',_> -leexinc.hrl:248: The pattern can never match the type <_,_,'error' | 'skip_token',_> -leexinc.hrl:253: The pattern can never match the type <_,_,'error',_> -leexinc.hrl:256: The pattern can never match the type <_,_,'error',_> -leexinc.hrl:260: Function yyrev/2 will never be called -yeccpre.hrl:49: Function return_error/2 will never be called -pokemon_pb.erl:47: The pattern <_, 'optional', 'undefined', _, _> can never match the type <1,'required',_,'string',[]> -pokemon_pb.erl:49: The pattern <_, 'repeated', 'undefined', _, _> can never match the type <1,'required',_,'string',[]> -pokemon_pb.erl:51: The pattern <_, 'repeated_packed', 'undefined', _, _> can never match the type <1,'required',_,'string',[]> -pokemon_pb.erl:52: The pattern <_, 'repeated_packed', [], _, _> can never match the type <1,'required',_,'string',[]> -pokemon_pb.erl:57: The pattern <_, 'repeated', [], _, Acc> can never match the type <1,'required',_,'string',[]> -pokemon_pb.erl:60: The pattern can never match the type <1,'required',_,'string',[]> -pokemon_pb.erl:63: The pattern can never match the type <1,'required',_,'string',[]> -pokemon_pb.erl:78: The call pokemon_pb:enum_to_int(Type::'string',Data::atom()) will never return since it differs in the 1st argument from the success typing arguments: ('pikachu','value') -pokemon_pb.erl:80: The pattern <'pikachu', 'value'> can never match the type <'string',atom()> -pokemon_pb.erl:102: The pattern 'true' can never match the type 'false' -pokemon_pb.erl:108: The pattern 'true' can never match the type 'false' -pokemon_pb.erl:118: The pattern 'true' can never match the type 'false' -pokemon_pb.erl:132: The pattern can never match the type <'-infinity' | 'false' | 'infinity' | 'nan' | 'true' | binary() | [byte()] | number(),'double' | 'int32'> -riak_core_pb.erl:47: The pattern <_, 'optional', 'undefined', _, _> can never match the type <1 | 2 | 3,'required',_,'bytes',[]> -riak_core_pb.erl:49: The pattern <_, 'repeated', 'undefined', _, _> can never match the type <1 | 2 | 3,'required',_,'bytes',[]> -riak_core_pb.erl:51: The pattern <_, 'repeated_packed', 'undefined', _, _> can never match the type <1 | 2 | 3,'required',_,'bytes',[]> -riak_core_pb.erl:52: The pattern <_, 'repeated_packed', [], _, _> can never match the type <1 | 2 | 3,'required',_,'bytes',[]> -riak_core_pb.erl:57: The pattern <_, 'repeated', [], _, Acc> can never match the type <1 | 2 | 3,'required',_,'bytes',[]> -riak_core_pb.erl:60: The pattern can never match the type <1 | 2 | 3,'required',_,'bytes',[]> -riak_core_pb.erl:63: The pattern can never match the type <1 | 2 | 3,'required',_,'bytes',[]> -riak_core_pb.erl:78: The call riak_core_pb:enum_to_int(Type::'bytes',Data::atom()) will never return since it differs in the 1st argument from the success typing arguments: ('pikachu','value') -riak_core_pb.erl:80: The pattern <'pikachu', 'value'> can never match the type <'bytes',atom()> -riak_core_pb.erl:102: The pattern 'true' can never match the type 'false' -riak_core_pb.erl:108: The pattern 'true' can never match the type 'false' -riak_core_pb.erl:118: The pattern 'true' can never match the type 'false' -riak_core_pb.erl:132: The pattern can never match the type <'-infinity' | 'false' | 'infinity' | 'nan' | 'true' | binary() | [byte()] | number(),'bytes'> -riakclient_pb.erl:51: The pattern <_, 'repeated_packed', 'undefined', _, _> can never match the type <1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9,'optional' | 'repeated' | 'required',_,'bool' | 'bytes' | 'rpbbucketprops' | 'rpbcontent' | 'rpblink' | 'rpbpair' | 'uint32',[binary() | [binary() | [any()]]]> -riakclient_pb.erl:52: The pattern <_, 'repeated_packed', [], _, _> can never match the type <1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9,'optional' | 'repeated' | 'required',_,'bool' | 'bytes' | 'rpbbucketprops' | 'rpbcontent' | 'rpblink' | 'rpbpair' | 'uint32',[binary() | [binary() | [any()]]]> -riakclient_pb.erl:63: The pattern can never match the type <1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9,'optional' | 'repeated' | 'required',_,'bool' | 'bytes' | 'rpbbucketprops' | 'rpbcontent' | 'rpblink' | 'rpbpair' | 'uint32',[binary() | [binary() | [any()]]]> -riakclient_pb.erl:78: The call riakclient_pb:enum_to_int(Type::'bool' | 'bytes' | 'rpbbucketprops' | 'rpbcontent' | 'rpblink' | 'rpbpair' | 'uint32',Data::atom()) will never return since it differs in the 1st argument from the success typing arguments: ('pikachu','value') -riakclient_pb.erl:80: The pattern <'pikachu', 'value'> can never match the type <'bool' | 'bytes' | 'rpbbucketprops' | 'rpbcontent' | 'rpblink' | 'rpbpair' | 'uint32',atom()> -riakclient_pb.erl:132: The pattern can never match the type <'-infinity' | 'false' | 'infinity' | 'nan' | 'true' | binary() | [byte()] | number(),'bool' | 'bytes' | 'rpbbucketprops' | 'rpbcontent' | 'rpblink' | 'rpbpair' | 'uint32'> diff --git a/eunit.log b/eunit.log new file mode 100644 index 000000000..89887af3f --- /dev/null +++ b/eunit.log @@ -0,0 +1,46 @@ +Eunit: basho_stats PASSED +Eunit: bear PASSED +Eunit: bitcask PASSED +Eunit: canola PASSED +Eunit: clique PASSED +Eunit: cluster_info PASSED +Eunit: cuttlefish PASSED +Eunit: ebloom PASSED +Eunit: eleveldb PASSED +Eunit: exometer_core PASSED +Eunit: folsom PASSED +Eunit: getopt PASSED +Eunit: goldrush PASSED +Eunit: hut PASSED +Eunit: hyper PASSED +Eunit: ibrowse PASSED +Eunit: kv_index_tictactree PASSED +Eunit: lager_syslog PASSED +Eunit: leveled PASSED +Eunit: lz4 PASSED +Eunit: meck PASSED +Eunit: mochiweb PASSED +Eunit: parse_trans PASSED +Eunit: pbkdf2 PASSED +Eunit: poolboy PASSED +Eunit: protobuffs PASSED +Eunit: ranch PASSED +Eunit: recon PASSED +Eunit: redbug PASSED +Eunit: riak_api PASSED +Eunit: riak_auth_mods PASSED +Eunit: riak_core PASSED +Eunit: riak_dt PASSED +Eunit: riak_ensemble PASSED +Eunit: riak_kv PASSED +Eunit: riak_pb PASSED +Eunit: riak_pipe PASSED +Eunit: riak_repl PASSED +Eunit: riak_sysmon PASSED +Eunit: riakc PASSED +Eunit: riakhttpc PASSED +Eunit: setup PASSED +Eunit: sext PASSED +Eunit: sidejob PASSED +Eunit: syslog PASSED +Eunit: webmachine PASSED diff --git a/priv/riak.schema b/priv/riak.schema index bb280b43a..8407c39d5 100644 --- a/priv/riak.schema +++ b/priv/riak.schema @@ -218,6 +218,12 @@ merge ]}. +%% VM emulator ignore break signal (prevent ^C / ^Gq) +{mapping, "erlang.vm.ignore_break_signal", "vm_args.+Bi", [ + {default, "true"}, + merge +]}. + {{#devrel}} %% Because of the 'merge' keyword in the proplist below, the docs and datatype %% are pulled from the leveldb schema. diff --git a/rebar.config b/rebar.config index 81d49c700..a0600d558 100644 --- a/rebar.config +++ b/rebar.config @@ -1,26 +1,26 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- {sub_dirs, ["rel", "apps/riak"]}. -{require_otp_vsn, "20"}. +{minimum_otp_vsn, "20.0"}. + {cover_enabled, true}. -%% TODO: nowarn_export_all on for now. -{erl_opts, [nowarn_export_all, - debug_info %% , fail_on_warning - ]}. -{eunit_opts, [verbose]}. +{erl_opts, [warnings_as_errors]}. +{eunit_opts, [nowarn_export_all, verbose]}. {deps, [ - {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {branch, "develop-3.0"}}}, - {cluster_info, {git, "git://github.com/basho/cluster_info.git", {branch, "develop-3.0"}}}, - {riak_kv, {git, "git://github.com/basho/riak_kv.git", {branch, "develop-3.0"}}}, - {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {branch, "develop-3.0"}}}, - {riak_repl, {git, "git://github.com/basho/riak_repl.git", {branch, "develop-3.0"}}}, - {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}} + {lager_syslog, {git, "git://github.com/basho/lager_syslog.git", {tag, "3.1.0"}}}, + {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}}, + {riak_kv, {git, "git://github.com/basho/riak_kv.git", {tag, "riak_kv-3.0.1"}}}, + {riak_auth_mods, {git, "git://github.com/basho/riak_auth_mods.git", {tag, "riak_kv-3.0.0"}}}, + + % {yokozuna, {git, "git://github.com/basho/yokozuna.git", {branch, "develop-3.0"}}}, + + {riak_repl, {git, "git://github.com/basho/riak_repl.git", {tag, "riak_kv-3.0.1"}}} ]}. {project_plugins, [ - {rebar3_cuttlefish, {git, "https://github.com/martincox/rebar3_cuttlefish", {branch, "fix/runner_dirs"}}} + {rebar3_cuttlefish, {git, "https://github.com/basho/rebar3_cuttlefish", {tag, "0.2.0"}}} ]}. {cuttlefish, [ @@ -38,7 +38,10 @@ bitcask_multi, eleveldb, eleveldb_multi, - multi_backend + leveled, + leveled_multi, + multi_backend, + riak_repl ]} ]}. @@ -46,6 +49,7 @@ [kernel, stdlib, lager, + lager_syslog, sasl, public_key, ssl, @@ -66,7 +70,7 @@ riak_api, riak_repl, cluster_info, - yokozuna, + % yokozuna, riak_auth_mods]}, {dev_mode, false}, @@ -84,7 +88,10 @@ {template, "rel/files/riak-chkconfig", "bin/riak-chkconfig"}, {template, "rel/files/riak-repl", "bin/riak-repl"}, + {template, "rel/files/riak", "usr/bin/riak"}, + {copy, "rel/files/check_ulimit", "bin/hooks/check_ulimit"}, + {copy, "rel/files/erl_maxlogsize", "bin/hooks/erl_maxlogsize"}, {copy, "rel/files/riak_not_running", "bin/hooks/riak_not_running"} ]}, @@ -96,17 +103,8 @@ {debug, "riak-debug"}, {repl, "riak-repl"}, {chkconfig, "riak-chkconfig"} - ]}, - + ]} - {extended_start_script_hooks, - [ {pre_start, - [{custom, "hooks/riak_not_running"}, - {custom, "hooks/check_ulimit"}]}, - {post_start, - [{wait_for_process, riak_core_node_watcher} - ]} - ]} ]}. {dialyzer, [{plt_apps, all_deps}]}. @@ -114,25 +112,63 @@ {profiles, [ {rel, [ {relx, [ - {overlay_vars, "rel/vars.config"} + {overlay_vars, "rel/vars.config"}, + {extended_start_script_hooks, + [{pre_start, + [{custom, "hooks/riak_not_running"}, + {custom, "hooks/check_ulimit"}, + {custom, "hooks/erl_maxlogsize"}]}, + {post_start, + [{wait_for_process, riak_core_node_watcher}]} + ]} ]} ]}, - {dev, [ - {relx, [ - {dev_mode, true} + {dev, [{relx, [ + {dev_mode, true}, + {extended_start_script_hooks, + [{pre_start, + [{custom, "hooks/riak_not_running"}, + {custom, "hooks/check_ulimit"}, + {custom, "hooks/erl_maxlogsize"}]}, + {post_start, + [{wait_for_process, riak_core_node_watcher}]} + ]} ]} ]}, {rpm, [ {relx, [ - {overlay_vars, "rpm.vars.config"}, + {overlay_vars, "rel/pkg/rpm/vars.config"}, {overlay, [ - {template, "rel/pkg/rpm/riak", "usr/bin/riak"} - ]} + {template, "rel/files/riak", "usr/sbin/riak"}, + {template, "rel/pkg/rpm/init.script", "etc/init.d/init.script"} + ]}, + {extended_start_script_hooks, + [{pre_start, + [{custom, "hooks/riak_not_running"}, + {custom, "hooks/check_ulimit"}, + {custom, "hooks/erl_maxlogsize"}]}, + {post_start, + [{pid, "/var/run/riak/riak.pid"}, + {wait_for_process, riak_core_node_watcher}]} + ]} ]} ]}, {deb, [ {relx, [ - {overlay_vars, "deb.vars.config"} + {overlay_vars, "debian/vars.config"}, + {overlay, [ + {template, "rel/files/riak", "usr/bin/riak"} + ]}, + {extended_start_script_hooks, + [{pre_start, + [{custom, "hooks/riak_not_running"}, + {custom, "hooks/check_ulimit"}, + {custom, "hooks/erl_maxlogsize"}]}, + {post_start, + [wait_for_vm_start, + {pid, "/run/riak/riak.pid"}, + {wait_for_process, riak_core_node_watcher}]} + ]} ]} ]} ]}. diff --git a/rebar.lock b/rebar.lock index 0cefc692f..624c0fcd3 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,12 +1,12 @@ {"1.1.0", [{<<"basho_stats">>, {git,"git://github.com/basho/basho_stats.git", - {ref,"176dee0537b5b3aff57646a7379d435ad990488e"}}, + {ref,"c5b7ecb92af7b050bc62d08c6fbf1aafd6bdeff0"}}, 2}, {<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},4}, {<<"bitcask">>, {git,"git://github.com/basho/bitcask.git", - {ref,"9184f763b135f9ff5061fa3ce1892cefa54cfe3a"}}, + {ref,"dd96f6dd84a1cad68ccc21959ba56b0426434d85"}}, 1}, {<<"canola">>, {git,"git://github.com/basho/canola.git", @@ -14,15 +14,15 @@ 1}, {<<"clique">>, {git,"git://github.com/basho/clique.git", - {ref,"dc282124f4371f5d0bb8514aea9d033d98d83dd3"}}, + {ref,"4014357e4e677164b890fdad08a45cfa54b3e8f3"}}, 2}, {<<"cluster_info">>, {git,"git://github.com/basho/cluster_info.git", - {ref,"18ed572102055626c1597339015d6501989df287"}}, + {ref,"389d43af7ac1550b3c01cd55b8147bcc0e20022f"}}, 0}, {<<"cuttlefish">>, {git,"https://github.com/basho/cuttlefish.git", - {ref,"1b4381e5f682d88a2f20ddb8511ff35098c40ea5"}}, + {ref,"e84070b6cf08196ece2ae54083eefae4dc4e4ecc"}}, 3}, {<<"ebloom">>, {git,"git://github.com/basho/ebloom.git", @@ -30,18 +30,16 @@ 1}, {<<"eleveldb">>, {git,"git://github.com/basho/eleveldb.git", - {ref,"5ca7491c947d24c764eb0a359370e83dcc0bb021"}}, + {ref,"3dee1dd7173507a4ad29df1ceb99cb70d6142478"}}, + 2}, + {<<"exometer_core">>, + {git,"git://github.com/Feuerlabs/exometer_core.git", + {ref,"547f28bc93c7cb3d3f1174fb4c510667a4ebb645"}}, 2}, - {<<"exometer_core">>,{pkg,<<"exometer_core">>,<<"1.5.7">>},2}, {<<"folsom">>,{pkg,<<"folsom">>,<<"0.8.7">>},3}, - {<<"fuse">>, - {git,"https://github.com/jlouis/fuse.git", - {ref,"21c6e52ced3af294f2fe636039106068da12eeeb"}}, - 1}, - {<<"gen_fsm_compat">>,{pkg,<<"gen_fsm_compat">>,<<"0.3.0">>},1}, {<<"getopt">>, - {git,"https://github.com/basho/getopt.git", - {ref,"880030ff0cbecb73ba75a78a7aa0cc1dcb7fe331"}}, + {git,"git://github.com/jcomellas/getopt.git", + {ref,"fbd210550677114a6c96cdf66e340eb248908d93"}}, 4}, {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},2}, {<<"hut">>,{pkg,<<"hut">>,<<"1.2.1">>},3}, @@ -51,15 +49,11 @@ 1}, {<<"ibrowse">>, {git,"git://github.com/basho/ibrowse.git", - {ref,"893864e25573b4ccd31a82b9412639751733ba40"}}, - 1}, + {ref,"aee5b93411b0a853494025005661623b13bf43f4"}}, + 2}, {<<"kv_index_tictactree">>, {git,"https://github.com/martinsumner/kv_index_tictactree.git", - {ref,"b2423eaf11493eb51fc8089a843f8c24d951404b"}}, - 1}, - {<<"kvc">>, - {git,"git://github.com/etrepum/kvc.git", - {ref,"5565fe51857747662410cc3c06362ebcf48a2f04"}}, + {ref,"fe623784cc3e3185238924bc6b1786f2d79edb36"}}, 1}, {<<"lager">>, {git,"git://github.com/erlang-lager/lager.git", @@ -67,28 +61,36 @@ 1}, {<<"lager_syslog">>, {git,"git://github.com/basho/lager_syslog.git", - {ref,"5c90466b25e5947020b09547e38439167e88363c"}}, + {ref,"152bb8eb254c225ebd8c579f9603f44b50df44ec"}}, 0}, {<<"leveled">>, {git,"https://github.com/martinsumner/leveled.git", - {ref,"432fe71bf09987f52fc407a776c23e8eca802451"}}, + {ref,"bf591a5aa988ef4c3699152d129fc81ca0896030"}}, 1}, {<<"lz4">>, - {git,"https://github.com/szktty/erlang-lz4", - {ref,"6bc5efe93bab327788bcde24374f16044b549dd8"}}, + {git,"https://github.com/martinsumner/erlang-lz4", + {ref,"d407af5de71303b8e381df718d2e5d05eb775c59"}}, 2}, + {<<"meck">>, + {git,"https://github.com/eproxus/meck.git", + {ref,"4ecc1ae9089edc6977e8c8c4cd41081513cc5590"}}, + 3}, {<<"mochiweb">>, - {git,"git://github.com/martinsumner/mochiweb.git", - {ref,"1c1d621632ce924b82bc733697d4115af91fc30f"}}, + {git,"git://github.com/basho/mochiweb.git", + {ref,"fb7bc696357b77fd8075de796d3687dfb5ed0d0b"}}, 2}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3}, {<<"pbkdf2">>, {git,"git://github.com/basho/erlang-pbkdf2.git", - {ref,"34b5f716fa3ec89ede36d009f7cc8c47bd019dd7"}}, + {ref,"681494c3ea03625c909a16b1b3ca295edf892057"}}, 2}, {<<"poolboy">>, - {git,"https://github.com/Kyorai/poolboy.git", - {ref,"58ae8431dcf9cc69045d618417e5e6e74c7e8dde"}}, + {git,"https://github.com/basho/poolboy.git", + {ref,"032ea3186fda78b4913deb8932af318ebb9b1258"}}, + 2}, + {<<"protobuffs">>, + {git,"https://github.com/basho/erlang_protobuffs.git", + {ref,"098efad8f85dfe556d64e2cf6ce31f2075808f67"}}, 2}, {<<"ranch">>, {git,"git://github.com/ninenines/ranch.git", @@ -104,44 +106,51 @@ 1}, {<<"riak_api">>, {git,"git://github.com/basho/riak_api.git", - {ref,"ada7e27a43ef3e98790df7b607f6dd0f0e4f13df"}}, + {ref,"0aec475c38ea17232b2e80d9c6c684a6ad9cd226"}}, 1}, {<<"riak_auth_mods">>, {git,"git://github.com/basho/riak_auth_mods.git", - {ref,"a991fd05929237413a641b60fc826412f98e099e"}}, + {ref,"1eccaef8e9e6200f0b75ea9312f0a731c8d572bf"}}, 0}, {<<"riak_core">>, {git,"git://github.com/basho/riak_core.git", - {ref,"b726b0b4c5bbef1a20b95d551d3e32f6dc8741b5"}}, + {ref,"3903efc8c975c47125babe06f2a4cc674616b658"}}, 1}, {<<"riak_dt">>, {git,"git://github.com/basho/riak_dt.git", - {ref,"cdf8cbf3009b7ecb4d659c1576a466cbefe47f55"}}, + {ref,"d0de4694f1423dc942844fbc494604a3c6df17d6"}}, 1}, {<<"riak_ensemble">>, {git,"https://github.com/basho/riak_ensemble", - {ref,"495833f7a79bb43e288a2a36c9acddfd27e8d90f"}}, + {ref,"573a271aa546dcdcaeaaf09b3de6949a3130e021"}}, 2}, {<<"riak_kv">>, {git,"git://github.com/basho/riak_kv.git", - {ref,"56cb3a1737c373fe4b09ca03681f36e56928a034"}}, + {ref,"d17495741ed001dcb54fa9e75e0dff026baaefde"}}, 0}, {<<"riak_pb">>, {git,"git://github.com/basho/riak_pb.git", - {ref,"55fab3a2642edf2293228d5a43748a0fc3c8e554"}}, + {ref,"64cabe25eec68acd6cc38035cb91814dff3c5f14"}}, 1}, {<<"riak_pipe">>, {git,"git://github.com/basho/riak_pipe.git", - {ref,"98944cb1b83fa83ab6011e3958252061f48e2172"}}, + {ref,"811ae70fc258e4c3e8d65ff0593ea06989b6dec1"}}, 1}, {<<"riak_repl">>, {git,"git://github.com/basho/riak_repl.git", - {ref,"8fcc415e89e867ee85a79949869b2497172360a4"}}, + {ref,"90c317286fdc8cff3709e8dcd6d99633d89201ce"}}, 0}, - {<<"riak_sysmon">>,{pkg,<<"riak_sysmon">>,<<"2.1.7">>},2}, + {<<"riak_sysmon">>, + {git,"git://github.com/basho/riak_sysmon.git", + {ref,"726df1f4c31108bb9366fb767b480f286e51f7fc"}}, + 2}, {<<"riakc">>, {git,"git://github.com/basho/riak-erlang-client", - {ref,"28b1349d2b9e0cf855baf8142a44815c116f39ad"}}, + {ref,"3c6c0d28507b87026f99b7de583eedda851338f7"}}, + 2}, + {<<"riakhttpc">>, + {git,"git://github.com/basho/riak-erlang-http-client", + {ref,"762c7cb5a1ca440207817dd452840431697430ab"}}, 1}, {<<"setup">>,{pkg,<<"setup">>,<<"2.0.2">>},3}, {<<"sext">>, @@ -150,7 +159,7 @@ 1}, {<<"sidejob">>, {git,"git://github.com/basho/sidejob.git", - {ref,"07dedd9b2043c9c6fd03b78fe8699f0f37418743"}}, + {ref,"d5e8f8450b92ab30610496ffb58f560d6a8bad6b"}}, 1}, {<<"syslog">>, {git,"git://github.com/Vagabond/erlang-syslog", @@ -159,20 +168,13 @@ {<<"webmachine">>, {git,"git://github.com/webmachine/webmachine.git", {ref,"92225b82fc702f78cf9d23248023ec841272df80"}}, - 2}, - {<<"yokozuna">>, - {git,"git://github.com/basho/yokozuna.git", - {ref,"328265b5beff5e65f5353cd708e3b6b4cc8363cd"}}, - 0}]}. + 2}]}. [ {pkg_hash,[ {<<"bear">>, <<"16264309AE5D005D03718A5C82641FCC259C9E8F09ADEB6FD79CA4271168656F">>}, - {<<"exometer_core">>, <<"AB97E34A5D69AB14E6AE161DB4CCA5B5E655E635B842F830EE6AB2CBFCFDC30A">>}, {<<"folsom">>, <<"A885F0AEEE4C84270954C88A55A5A473D6B2C7493E32FFDC5765412DD555A951">>}, - {<<"gen_fsm_compat">>, <<"5903549F67D595F58A7101154CBE0FDD46955FBFBE40813F1E53C23A970FF5F4">>}, {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>}, {<<"hut">>, <<"08D46679523043424870723923971889E8A34D63B2F946A35B46CF921D1236E7">>}, {<<"parse_trans">>, <<"09765507A3C7590A784615CFD421D101AEC25098D50B89D7AA1D66646BC571C1">>}, - {<<"riak_sysmon">>, <<"AF420DF0F7569E1F12BCD465745164CB6189EB93F118D5CDB3F90FEB3F8BF47D">>}, {<<"setup">>, <<"1203F4CDA11306C2E34434244576DED0A7BBFB0908D9A572356C809BD0CDF085">>}]} ]. diff --git a/rebar3 b/rebar3 index 77e0daeaa..e550663ab 100755 Binary files a/rebar3 and b/rebar3 differ diff --git a/rel/files/advanced.config b/rel/files/advanced.config index cc2e6feee..cd3c19fa8 100644 --- a/rel/files/advanced.config +++ b/rel/files/advanced.config @@ -100,5 +100,9 @@ ] } ] -} +}, +{setup, + [ + {home, "{{platform_data_dir}}/setup"} + ]} ]. diff --git a/rel/files/erl_maxlogsize b/rel/files/erl_maxlogsize new file mode 100644 index 000000000..395a9e865 --- /dev/null +++ b/rel/files/erl_maxlogsize @@ -0,0 +1,3 @@ +#!/bin/sh +RUN_ERL_LOG_MAXSIZE="${RUN_ERL_LOG_MAXSIZE:=10000000}" +export RUN_ERL_LOG_MAXSIZE \ No newline at end of file diff --git a/rel/files/riak b/rel/files/riak new file mode 100755 index 000000000..b41b97c2f --- /dev/null +++ b/rel/files/riak @@ -0,0 +1,49 @@ +#!/bin/bash + +RUNNER_GEN_DIR="${RUNNER_GEN_DIR:-{{ platform_gen_dir }}}" +RELEASE_ROOT_DIR="${RELEASE_ROOT_DIR:-{{ runner_base_dir }}}" +PID_DIR={{pid_dir}} +COMMAND={{platform_bin_dir}}/riak +RUNNER_LOG_DIR={{platform_log_dir}} +RELX_CONFIG_PATH=${RUNNER_GEN_DIR}/sys.config +VMARGS_PATH=${RUNNER_GEN_DIR}/vm.args + + +# On first running the sys.config and vm.args will not be a link +# as cfconfig has not yet been run as a pre_start hook. If there's no +# link use the default for now +if [ ! -L $RELX_CONFIG_PATH ]; then + cp $RELEASE_ROOT_DIR/releases/{{release_version}}/sys.config $RELX_CONFIG_PATH +fi +if [ ! -L $VMARGS_PATH ]; then + cp $RELEASE_ROOT_DIR/releases/{{release_version}}/vm.args $VMARGS_PATH +fi + +# When running as a service, running as riak not as root, and systemd has created PID folder +if [[ $EUID -ne 0 ]]; then + case "$1" in + start|console|foreground) + RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches + ;; + *) + RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} + ;; + esac +else + # In this case we're running sudo riak - so have root access, but cannot rely + # systemd having created the PID dir, and need to sudo to the riak user + if [ ! -d $PID_DIR ]; then + mkdir $PID_DIR + chown riak:riak $PID_DIR + fi + case "$1" in + start|console|foreground) + su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*} -pa {{platform_lib_dir}}/patches" + ;; + *) + su - riak -c "RELX_CONFIG_PATH=${RELX_CONFIG_PATH} VMARGS_PATH=${VMARGS_PATH} RUNNER_LOG_DIR=${RUNNER_LOG_DIR} PIPE_DIR=${PIPE_DIR} ${COMMAND} ${*}" + ;; + esac +fi + + diff --git a/rel/files/riak-admin b/rel/files/riak-admin index 1b1b2657e..6f4cdc2ce 100755 --- a/rel/files/riak-admin +++ b/rel/files/riak-admin @@ -9,7 +9,7 @@ ORIGINAL_DIR=$(pwd) cd $RUNNER_BASE_DIR # Identify the script name -SCRIPT=`basename $0` +SCRIPT="riak-admin" usage() { echo "Usage: $SCRIPT { cluster | join | leave | backup | restore | test | " @@ -281,15 +281,14 @@ cluster_admin() relx_nodetool rpc riak_core_console clear_staged ;; status) - echo "script: $SCRIPT / $@" - relx_nodetool rpc riak_core_console command "$SCRIPT-admin" cluster $@ + relx_nodetool rpc riak_core_console command $SCRIPT cluster $@ ;; partitions|partition) - relx_nodetool rpc riak_core_console command "$SCRIPT-admin" cluster $@ + relx_nodetool rpc riak_core_console command $SCRIPT cluster $@ ;; partition[_-]count) shift - relx_nodetool rpc riak_core_console command "$SCRIPT-admin" cluster partition-count $@ + relx_nodetool rpc riak_core_console command $SCRIPT cluster partition-count $@ ;; *) echo "\ diff --git a/rel/pkg/Makefile b/rel/pkg/Makefile index 81bf3c246..95658db8d 100644 --- a/rel/pkg/Makefile +++ b/rel/pkg/Makefile @@ -83,7 +83,7 @@ PKG_BUILD ?= 1 ## Call platform dependent makefile ostype: varcheck $(if $(PKGERDIR),,$(error "Operating system '$(OS)' not supported by node_package")) - $(MAKE) -C $(PKGERDIR) -f Makefile + $(MAKE) -C $(BASE_DIR)/rel/pkg/out/riak-$(PKG_ID) -f $(BASE_DIR)/rel/pkg/$(PKGERDIR)/Makefile ## Check required settings before continuing varcheck: diff --git a/rel/pkg/deb/Makefile b/rel/pkg/deb/Makefile index 79fda8215..780de4a9a 100644 --- a/rel/pkg/deb/Makefile +++ b/rel/pkg/deb/Makefile @@ -1,19 +1,27 @@ +export + +TAR_VERSION = $(shell git describe --tags | sed -e 's/\([0-9.]*\-[0-9]*\)-.*/\1/') default: - ln -sf $(PKG_ID).tar.gz ../{{package_name}}_$(PKG_VERSION).orig.tar.gz - export DEBFULLNAME="{{vendor_contact_name}}"; \ - export DEBEMAIL="{{vendor_contact_email}}"; \ - dch --create --package {{package_name}} -v "$(PKG_VERSION)-$(PKG_BUILD)" \ - "Build from $(PKG_VERSION)";\ + mkdir -p $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/debian + cp -R $(BASE_DIR)/rel/pkg/deb/debian/* $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/debian + + tar -xf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz -C $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/ + ln -sf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz $(BASE_DIR)/rel/pkg/out/$(TAR_VERSION).orig.tar.gz + + cd $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/; \ + echo $(shell pwd); \ + dch --create --package riak -v "$(PKG_VERSION)" \ + "Build from $(PKG_ID)";\ debuild --prepend-path=$(ERLANG_BIN) \ - -e REVISION=$(PKG_VERSION) \ - -e RELEASE=$(PKG_BUILD) \ - -e REBAR=$(REBAR) \ - {{debuild_extra_options}} \ - -uc -us - mkdir -p ../packages - cd .. && mv *$(PKG_VERSION)-$(PKG_BUILD)_*.deb packages - cd ../packages && \ + -e REVISION=$(PKG_VERSION) \ + -e RELEASE=$(PKG_BUILD) \ + -e REBAR=$(REBAR) \ + -i -uc -us -b + + mkdir -p $(BASE_DIR)/rel/pkg/packages + cd $(BASE_DIR)/rel/pkg/out && mv *.deb ../packages + cd $(BASE_DIR)/rel/pkg/packages && \ for debfile in *.deb; do \ sha256sum $${debfile} > $${debfile}.sha \ ; done diff --git a/rel/pkg/deb/control b/rel/pkg/deb/control deleted file mode 100644 index 37090553f..000000000 --- a/rel/pkg/deb/control +++ /dev/null @@ -1,16 +0,0 @@ -Source: {{package_name}} -Section: net -Priority: extra -Maintainer: {{vendor_contact_name}} <{{vendor_contact_email}}> -Build-Depends: debhelper (>= 7) -Standards-Version: 3.9.3 -Homepage: {{vendor_url}} - -Package: {{package_name}} -Architecture: any -Depends: ${misc:Depends}, ${shlibs:Depends}, adduser, logrotate, sudo, {{deb_depends}} -Homepage: {{vendor_url}} -Description: {{package_shortdesc}} - {{package_desc}} -{{package_replacement_line}} -{{package_conflicts_line}} diff --git a/rel/pkg/deb/copyright b/rel/pkg/deb/copyright deleted file mode 100644 index d02cb9889..000000000 --- a/rel/pkg/deb/copyright +++ /dev/null @@ -1,8 +0,0 @@ -Format: http://dep.debian.net/deps/dep5 -Upstream-Name: {{package_name}} -Upstream-Contact: {{vendor_contact_name}} <{{vendor_contact_email}}> - -Files: * -Copyright: {{copyright}} -License: {{license_type}} - {{license_full_text}} diff --git a/rel/pkg/deb/deb.template b/rel/pkg/deb/deb.template deleted file mode 100644 index 1f0c07c13..000000000 --- a/rel/pkg/deb/deb.template +++ /dev/null @@ -1,41 +0,0 @@ -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% -{variables, [ - {package_name, "package_name"}, - {package_install_name, "package_install_name"}, - {package_install_user, "package_install_user"}, - {package_install_user_desc, "package_install_user_desc"}, - {package_install_group, "package_install_group"}, - {package_replacement_line, "{{package_replacement_line_debian}}"}, - {package_conflicts_line, "{{package_conflicts_line_debian}}"}, - {vendor_name, "vendor_name"}, - {vendor_url, "vendor_url"}, - {vendor_contact_name, "vendor_contact_name"}, - {vendor_contact_email, "vendor_contact_email"}, - {copyright, "copyright"}, - {license_type, "license_type"}, - {license_full_text, ""}, - {bin_or_sbin, "bin"}, - - %% Platform Specific Settings - {platform_bin_dir, "/usr/{{bin_or_sbin}}"}, - {platform_data_dir, "/var/lib/{{package_install_name}}"}, - {platform_etc_dir, "/etc/{{package_install_name}}"}, - {platform_base_dir, "/usr/lib/{{package_install_name}}"}, - {platform_lib_dir, "/usr/lib/{{package_install_name}}/lib"}, - {platform_log_dir, "/var/log/{{package_install_name}}"} - ] -}. -{template, "Makefile", "Makefile"}. -{template, "compat", "compat"}. -{template, "control", "control"}. -{template, "copyright", "copyright"}. -{template, "dirs", "dirs"}. -{template, "install", "install"}. -{template, "postinst", "postinst"}. -{template, "postrm", "postrm"}. -{template, "rules", "rules"}. -{template, "vars.config", "vars.config"}. -{template, "init.script", "{{package_name}}.{{package_install_name}}.init"}. -{template, "package.service", "{{package_name}}.{{package_install_name}}.service"}. -{template, "package.manpages", "{{package_name}}.manpages"}. diff --git a/rel/pkg/deb/compat b/rel/pkg/deb/debian/compat similarity index 100% rename from rel/pkg/deb/compat rename to rel/pkg/deb/debian/compat diff --git a/rel/pkg/deb/debian/control b/rel/pkg/deb/debian/control new file mode 100644 index 000000000..49244e1be --- /dev/null +++ b/rel/pkg/deb/debian/control @@ -0,0 +1,13 @@ +Source: riak +Section: net +Priority: extra +Maintainer: packaging +Build-Depends: debhelper (>= 7) +Standards-Version: 3.9.3 +Homepage: riak.com + +Package: riak +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, adduser, logrotate, sudo +Homepage: riak.com +Description: Riak KV Database diff --git a/rel/pkg/deb/debian/copyright b/rel/pkg/deb/debian/copyright new file mode 100644 index 000000000..28d10fb6d --- /dev/null +++ b/rel/pkg/deb/debian/copyright @@ -0,0 +1,4 @@ +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: riak +Upstream-Contact: packaging@riak.com + diff --git a/rel/pkg/deb/debian/dirs b/rel/pkg/deb/debian/dirs new file mode 100644 index 000000000..588cf1aa7 --- /dev/null +++ b/rel/pkg/deb/debian/dirs @@ -0,0 +1,2 @@ +etc/logrotate.d +var/log/riak diff --git a/rel/pkg/deb/debian/install b/rel/pkg/deb/debian/install new file mode 100644 index 000000000..18904d00f --- /dev/null +++ b/rel/pkg/deb/debian/install @@ -0,0 +1,8 @@ +rel/riak/lib usr/lib/riak +rel/riak/share usr/lib/riak +rel/riak/erts* usr/lib/riak +rel/riak/releases usr/lib/riak +rel/riak/bin usr/lib/riak +rel/riak/etc/* etc/riak +rel/riak/data/* var/lib/riak +rel/riak/usr/bin/* usr/sbin diff --git a/rel/pkg/deb/postinst b/rel/pkg/deb/debian/postinst similarity index 51% rename from rel/pkg/deb/postinst rename to rel/pkg/deb/debian/postinst index 88f272a5f..35d487a9f 100755 --- a/rel/pkg/deb/postinst +++ b/rel/pkg/deb/debian/postinst @@ -1,26 +1,26 @@ #!/bin/sh -# postinst script for {{package_name}} +# postinst script for Riak # # see: dh_installdeb(1) set -e # create group -if ! getent group {{package_install_group}} >/dev/null; then - addgroup --system {{package_install_group}} +if ! getent group riak >/dev/null; then + addgroup --system riak fi # create user -if ! getent passwd {{package_install_user}} >/dev/null; then - adduser --ingroup {{package_install_group}} \ - --home /var/lib/{{package_install_name}} \ +if ! getent passwd riak >/dev/null; then + adduser --ingroup riak \ + --home /var/lib/riak \ --disabled-password \ --system --shell /bin/bash --no-create-home \ - --gecos "{{package_install_user_desc}}" {{package_install_user}} + --gecos "Riak KV Database" riak fi for i in lib log; do - chown -R {{package_install_user}}:{{package_install_group}} /var/$i/{{package_install_name}} + chown -R riak:riak /var/$i/riak done case "$1" in diff --git a/rel/pkg/deb/postrm b/rel/pkg/deb/debian/postrm similarity index 54% rename from rel/pkg/deb/postrm rename to rel/pkg/deb/debian/postrm index df83a1fb8..2fc8de5b5 100755 --- a/rel/pkg/deb/postrm +++ b/rel/pkg/deb/debian/postrm @@ -1,5 +1,5 @@ #!/bin/sh -# postrm script for {{package_name}} +# postrm script for Riak # # see: dh_installdeb(1) @@ -21,26 +21,26 @@ set -e case "$1" in purge) - rm -f /etc/default/{{package_install_name}} - if [ -d /var/log/{{package_install_name}} ]; then - rm -r /var/log/{{package_install_name}} + rm -f /etc/default/riak + if [ -d /var/log/riak ]; then + rm -r /var/log/riak fi - if [ -d /var/run/{{package_install_name}} ]; then - rm -r /var/run/{{package_install_name}} + if [ -d /var/run/riak ]; then + rm -r /var/run/riak fi - if [ -d /etc/{{package_install_name}} ]; then - rm -r /etc/{{package_install_name}} + if [ -d /etc/riak ]; then + rm -r /etc/riak fi - if [ -e /etc/init.d/{{package_install_name}} ]; then - rm /etc/init.d/{{package_install_name}} + if [ -e /etc/init.d/riak ]; then + rm /etc/init.d/riak fi # Remove User & Group, killing any process owned by them - if getent passwd {{package_install_user}} >/dev/null; then - pkill -u {{package_install_user}} || true - deluser --quiet --system {{package_install_user}} + if getent passwd riak >/dev/null; then + pkill -u riak || true + deluser --quiet --system riak fi - if getent group {{package_install_group}} >/dev/null; then - delgroup --quiet --system --only-if-empty {{package_install_group}} || true + if getent group riak >/dev/null; then + delgroup --quiet --system --only-if-empty riak || true fi ;; diff --git a/rel/pkg/deb/package.manpages b/rel/pkg/deb/debian/riak.manpages similarity index 100% rename from rel/pkg/deb/package.manpages rename to rel/pkg/deb/debian/riak.manpages diff --git a/rel/pkg/deb/init.script b/rel/pkg/deb/debian/riak.riak.init similarity index 88% rename from rel/pkg/deb/init.script rename to rel/pkg/deb/debian/riak.riak.init index c0fbd9055..3405ea0e6 100755 --- a/rel/pkg/deb/init.script +++ b/rel/pkg/deb/debian/riak.riak.init @@ -1,16 +1,16 @@ #! /bin/bash ### BEGIN INIT INFO -# Provides: {{package_name}} +# Provides: riak # Required-Start: $remote_fs $syslog # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: {{package_shortdesc}} -# Description: {{package_desc}} +# Short-Description: Riak KV Database +# Description: Riak KV Database ### END INIT INFO -NAME={{package_install_name}} -DAEMON=/usr/{{bin_or_sbin}}/$NAME +NAME=riak +DAEMON=/usr/sbin/$NAME SCRIPTNAME=/etc/init.d/$NAME RUNDIR=/var/run/$NAME @@ -25,7 +25,7 @@ RUNDIR=/var/run/$NAME # if no HOME was set in /etc/default/$NAME then set one here # to the data directory for erlexec's sake if [ -z "$HOME" ]; then - export HOME={{platform_data_dir}} + export HOME="/var/lib/riak" fi # @@ -35,7 +35,7 @@ do_start() { if [ ! -d $RUNDIR ]; then mkdir $RUNDIR - chown {{package_install_user}}:{{package_install_group}} $RUNDIR + chown riak:riak $RUNDIR fi # Return # 0 if daemon has been started @@ -44,8 +44,8 @@ do_start() # Startup with the appropriate user start-stop-daemon --start \ - --name {{package_install_name}} \ - --user {{package_install_user}} \ + --name riak \ + --user riak \ --exec $DAEMON -- start \ || return 2 } @@ -70,7 +70,7 @@ do_stop() start-stop-daemon --stop \ --quiet \ --retry=TERM/30/KILL/5 \ - --user {{package_install_user}} \ + --user riak \ --exec $ERTS_PATH/run_erl return $? } diff --git a/rel/pkg/deb/debian/riak.riak.service b/rel/pkg/deb/debian/riak.riak.service new file mode 100644 index 000000000..4fa022264 --- /dev/null +++ b/rel/pkg/deb/debian/riak.riak.service @@ -0,0 +1,14 @@ +[Unit] +Description=Riak KV Database + +[Service] +User=riak +ExecStart=/usr/sbin/riak start +ExecStop=/usr/sbin/riak stop +Type=forking +PIDFile=/run/riak/riak.pid +EnvironmentFile=-/etc/default/riak +RuntimeDirectory=riak + +[Install] +WantedBy=multi-user.target diff --git a/rel/pkg/deb/rules b/rel/pkg/deb/debian/rules similarity index 70% rename from rel/pkg/deb/rules rename to rel/pkg/deb/debian/rules index d06fee81f..082a39c12 100755 --- a/rel/pkg/deb/rules +++ b/rel/pkg/deb/debian/rules @@ -11,14 +11,14 @@ # Uncomment this to turn on verbose mode. export DH_VERBOSE=1 -ROOTDIR := debian/{{package_name}} +ROOTDIR := debian/riak ## Clear variables that may confound our build of sub-projects; also ## note that it is necessary to use overlay_vars relative to .. as ## the generate command EXECUTES in rel/ build: unset CC CFLAGS CPPFLAGS LDFLAGS CXX CXXFLAGS \ - && OVERLAY_VARS="overlay_vars=../debian/vars.config" make rel + && make rel-deb touch build clean: @@ -28,7 +28,7 @@ clean: ## dh_shlibdeps was added to figure out the dependencies on shared libraries ## and will populate the ${shlibs:Depends} callout in the control file -install: LIBDIR := $(ROOTDIR)/usr/lib/{{package_install_name}} +install: LIBDIR := $(ROOTDIR)/usr/lib/riak install: build dh_testdir dh_testroot @@ -36,16 +36,11 @@ install: build dh_installdirs dh_install dh_installman - dh_installinit --name={{package_install_name}} --no-start + dh_installinit --name=riak --no-start dh_fixperms - chmod 0755 $(ROOTDIR)/etc/{{package_install_name}} - chmod -R a+rX $(ROOTDIR)/etc/{{package_install_name}} - for file in lib/env.sh lib/app_epath.sh erts-*/bin/nodetool; do \ - if [ -f $(LIBDIR)/$$file ]; then \ - chmod 0755 $(LIBDIR)/$$file; \ - fi; \ - done - chmod -R go+rX debian/{{package_name}}/usr/lib/{{package_install_name}}/lib/ + chmod 0755 $(ROOTDIR)/etc/riak + chmod -R a+rX $(ROOTDIR)/etc/riak + chmod -R go+rX debian/riak/usr/lib/riak/lib/ dh_shlibdeps # We have nothing to do by default. diff --git a/rel/pkg/deb/debian/vars.config b/rel/pkg/deb/debian/vars.config new file mode 100644 index 000000000..8f8fa0827 --- /dev/null +++ b/rel/pkg/deb/debian/vars.config @@ -0,0 +1,60 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et + +%% Platform-specific installation paths +{platform_bin_dir, "/usr/lib/riak/bin"}. +{platform_data_dir, "/var/lib/riak"}. +{platform_etc_dir, "/etc/riak"}. +{platform_base_dir, "/usr/lib/riak"}. +{platform_lib_dir, "/usr/lib/riak/lib"}. +{platform_log_dir, "/var/log/riak"}. +{platform_gen_dir, "/var/lib/riak"}. + +{runner_script_dir, "/usr/lib/riak/bin"}. +{runner_base_dir, "/usr/lib/riak"}. +{runner_etc_dir, "/etc/riak"}. +{runner_log_dir, "/var/log/riak"}. +{runner_user, "riak"}. +{runner_lib_dir, "/usr/lib/riak/lib"}. +{runner_patch_dir, "/usr/lib/riak/lib/patches"}. +{pipe_dir, "/tmp/riak/"}. +{pid_dir, "/run/riak/"}. +{package_replacement_line, ""}. +{package_conflicts_line, ""}. + +{cluster_manager_ip, "127.0.0.1"}. +{cluster_manager_port, 9080}. + +{web_ip, "127.0.0.1"}. +{web_port, 8098}. +{handoff_ip, "0.0.0.0"}. +{handoff_port, 8099}. +{pb_ip, "127.0.0.1"}. +{pb_port, 8087}. + +{storage_backend, "bitcask"}. + +{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. +{sasl_log_dir, "{{platform_log_dir}}/sasl"}. +{repl_data_root, "{{platform_data_dir}}/riak_repl/"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +%% +%% cuttlefish +%% +{cuttlefish, "on"}. +{cuttlefish_conf, "riak.conf"}. + +%% +%% yokozuna +%% +{yz_solr_port, 8093}. +{yz_solr_jmx_port, 8985}. + +%% lager +{console_log_default, file}. + +%% +%% etc/vm.args +%% +{node, "riak@127.0.0.1"}. diff --git a/rel/pkg/deb/dirs b/rel/pkg/deb/dirs deleted file mode 100644 index 3220e0d31..000000000 --- a/rel/pkg/deb/dirs +++ /dev/null @@ -1,2 +0,0 @@ -etc/logrotate.d -var/log/{{package_install_name}} diff --git a/rel/pkg/deb/install b/rel/pkg/deb/install deleted file mode 100644 index 5f314b5f2..000000000 --- a/rel/pkg/deb/install +++ /dev/null @@ -1,10 +0,0 @@ -rel/{{package_install_name}}/lib usr/lib/{{package_install_name}} -rel/{{package_install_name}}/erts* usr/lib/{{package_install_name}} -rel/{{package_install_name}}/releases usr/lib/{{package_install_name}} - -{{#package_commands}} -rel/{{package_install_name}}/bin/{{name}} usr/{{bin_or_sbin}} -{{/package_commands}} - -rel/{{package_install_name}}/etc/* etc/{{package_install_name}} -rel/{{package_install_name}}/data/* var/lib/{{package_install_name}} \ No newline at end of file diff --git a/rel/pkg/deb/package.service b/rel/pkg/deb/package.service deleted file mode 100644 index 751414252..000000000 --- a/rel/pkg/deb/package.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description={{package_shortdesc}} - -[Service] -ExecStart=/usr/{{bin_or_sbin}}/{{package_install_name}} start -ExecStop=/usr/{{bin_or_sbin}}/{{package_install_name}} stop -User={{package_install_user}} -Type=forking -PIDFile=/var/run/{{package_install_name}}/{{package_install_name}}.pid -EnvironmentFile=-/etc/default/{{package_install_name}} -RuntimeDirectory={{package_install_name}} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/rel/pkg/deb/vars.config b/rel/pkg/deb/vars.config deleted file mode 100644 index 6e3f56ed4..000000000 --- a/rel/pkg/deb/vars.config +++ /dev/null @@ -1,21 +0,0 @@ -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et - -%% Platform-specific installation paths -{platform_bin_dir, "{{platform_bin_dir}}"}. -{platform_data_dir, "{{platform_data_dir}}"}. -{platform_etc_dir, "{{platform_etc_dir}}"}. -{platform_base_dir, "{{platform_base_dir}}"}. -{platform_lib_dir, "{{platform_lib_dir}}"}. -{platform_log_dir, "{{platform_log_dir}}"}. - -{runner_script_dir, "{{platform_bin_dir}}"}. -{runner_base_dir, "{{platform_base_dir}}"}. -{runner_etc_dir, "{{platform_etc_dir}}"}. -{runner_log_dir, "{{platform_log_dir}}"}. -{runner_user, "{{package_install_user}}"}. -{runner_lib_dir, "{{platform_lib_dir}}"}. -{runner_patch_dir, "{{platform_lib_dir}}/{{package_patch_dir}}"}. -{pipe_dir, "/tmp/{{package_install_name}}/"}. -{package_replacement_line, "{{package_replacement_line_debian}}"}. -{package_conflicts_line, "{{package_replacement_line_debian}}"}. diff --git a/rel/pkg/rpm/Makefile b/rel/pkg/rpm/Makefile index 19e50e426..fed36bf7a 100644 --- a/rel/pkg/rpm/Makefile +++ b/rel/pkg/rpm/Makefile @@ -15,9 +15,9 @@ default: --define "_revision $(PKG_VERSION)" \ --define "_version $(PKG_VERSION_NO_H)" \ --define "_release $(PKG_BUILD)" \ - --define "_tarname riak-$(PKG_ID).tar.gz" \ - --define "_tarname_base riak-$(PKG_ID)" \ - -ba specfile + --define "_tarname $(PKG_ID).tar.gz" \ + --define "_tarname_base $(PKG_ID)" \ + -ba $(BASE_DIR)/rel/pkg/rpm/specfile cd $(BASE_DIR)/rel/pkg/out/packages && \ for rpmfile in *.rpm; do \ sha256sum $${rpmfile} > $${rpmfile}.sha \ diff --git a/rel/pkg/rpm/init.script b/rel/pkg/rpm/init.script new file mode 100755 index 000000000..ec184dff3 --- /dev/null +++ b/rel/pkg/rpm/init.script @@ -0,0 +1,149 @@ +#!/bin/sh +# +# riak +# +# chkconfig: 2345 80 30 +# description: Riak KV Database +# processname: beam +# + +# Source function library. +. /etc/rc.d/init.d/functions + +RETVAL=0 +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Riak KV Database" +NAME=riak +DAEMON=/usr/sbin/$NAME +lockfile=/var/lock/subsys/$NAME +pidfile=/var/run/$NAME/$NAME.pid + +# Check for script, config and data dirs +[ -x /usr/sbin/$NAME ] || exit 0 +[ -d /etc/$NAME ] || exit 0 +[ -d /var/lib/$NAME ] || exit 0 + +# Read configuration variable file if it is present and readable +[ -r /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME + +# `service` strips all environmental VARS so +# if no HOME was set in /etc/sysconfig/$NAME then set one here +# to the data directory for erlexec's sake +if [ -z "$HOME" ]; then + export HOME={{platform_data_dir}} +fi + +status -p $pidfile -l $(basename $lockfile) $NAME >/dev/null 2>&1 +running=$? + +check_pid_status() { + pid=$(ps ax | grep beam.smp | grep "\-progname $NAME " | awk '{print $1}') + if [ "$pid" = "" ]; then + # prog not running? + return 1 + else + # running + return 0 + fi +} + +start() { + # Start daemons. + echo -n $"Starting riak: " + $DAEMON start + RETVAL=$? + if [ $RETVAL -eq 0 ]; then + touch $lockfile + success + else + failure $"$NAME start" + fi + echo + return $RETVAL +} + +stop() { + # Stop daemon. + echo -n $"Shutting down riak: " + $DAEMON stop 2>/dev/null + for n in $(seq 1 10); do + sleep 1 + check_pid_status + RETVAL=$? + if [ $RETVAL -eq 1 ]; then + break + fi + done + if [ $RETVAL -eq 1 ]; then + rm -f $lockfile $pidfile + success + echo && return 0 + else + failure $"$NAME stop" + echo && return 1 + fi +} + +hardstop() { + echo -n $"Shutting down $NAME: " + su - riak -c "ps -ef | grep beam.smp | grep '\-progname $NAME ' | grep -v grep | awk '{print \$2}' | xargs kill -9" + for n in $(seq 1 10); do + sleep 1 + check_pid_status + RETVAL=$? + if [ $RETVAL -eq 1 ]; then + break + fi + done + if [ $RETVAL -eq 1 ]; then + rm -f $lockfile $pidfile + success + echo && return 0 + else + failure $"$NAME hardstop" + echo && return 1 + fi +} + +# See how we were called. +case "$1" in + start) + [ $running -eq 0 ] && exit 0 + start + ;; + stop) + if [ $running -eq 0 ]; then + stop + else + check_pid_status + RETVAL=$? + if [ $RETVAL -eq 1 ]; then + rm -f $lockfile $pidfile + fi + exit 0 + fi + ;; + restart|force-reload) + [ $running -eq 0 ] && stop + start + ;; + hardstop) + [ $running -eq 0 ] || exit 0 + hardstop + ;; + condrestart|try-restart) + [ $running -eq 0 ] || exit 0 + restart + ;; + status) + status -p $pidfile -l $(basename $lockfile) $NAME + ;; + ping) + $DAEMON ping || exit $? + ;; + *) + echo $"Usage: $0 {start|stop|restart|force-reload|hardstop|condrestart|try-restart|status|ping}" + exit 1 +esac + +exit $? diff --git a/rel/pkg/rpm/specfile b/rel/pkg/rpm/specfile index fd9a07412..05ab83604 100644 --- a/rel/pkg/rpm/specfile +++ b/rel/pkg/rpm/specfile @@ -64,7 +64,7 @@ EOF make rel-rpm %install -%define relpath %{_builddir}/%{buildsubdir}/_build/rel+rpm/rel/riak +%define relpath %{_builddir}/%{buildsubdir}/_build/rpm/rel/riak %define buildroot_lib %{buildroot}%{_libdir}/riak %define buildroot_etc %{buildroot}%{_sysconfdir}/riak %define buildroot_bin %{buildroot_lib}/bin @@ -72,7 +72,7 @@ make rel-rpm mkdir -p %{buildroot_etc} mkdir -p %{buildroot_lib} mkdir -p %{buildroot_bin} -mkdir -p %{buildroot}%{_bindir} +mkdir -p %{buildroot}%{_sbindir} mkdir -p %{buildroot}%{_localstatedir}/lib/riak mkdir -p %{buildroot}%{_localstatedir}/log/riak @@ -87,13 +87,17 @@ if [ -d %{relpath}/bin ]; then \ cd - fi -cp %{relpath}/usr/bin/riak %{buildroot}%{_bindir} -cp -R %{relpath}/etc/* %{buildroot_etc} +cp %{relpath}/usr/sbin/riak %{buildroot}%{_sbindir} +cp %{relpath}/etc/riak.conf %{buildroot_etc} +cp %{relpath}/etc/advanced.config %{buildroot_etc} mkdir -p %{buildroot}%{_localstatedir}/lib/riak cp -R %{relpath}/data/* \ %{buildroot}%{_localstatedir}/lib/riak +mkdir -p %{buildroot}%{_sysconfdir}/init.d +install -m755 %{relpath}/etc/init.d/init.script %{buildroot}%{_sysconfdir}/init.d/riak + # Needed to work around check-rpaths which seems to be hardcoded into recent # RPM releases export QA_RPATHS=3 @@ -142,9 +146,10 @@ exit 0 %{_localstatedir}/log/riak %{_libdir}/* %defattr(-,root,root) +%{_sysconfdir}/init.d/riak %dir %{_sysconfdir}/riak -/usr/bin/* +/usr/sbin/* %config(noreplace) %{_sysconfdir}/riak/* %clean -#rm -rf %{buildroot} +rm -rf %{buildroot} diff --git a/rel/pkg/rpm/vars.config b/rel/pkg/rpm/vars.config new file mode 100644 index 000000000..c2c590e02 --- /dev/null +++ b/rel/pkg/rpm/vars.config @@ -0,0 +1,60 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et + +%% Platform-specific installation paths +{platform_bin_dir, "/usr/lib64/riak/bin"}. +{platform_data_dir, "/var/lib/riak"}. +{platform_etc_dir, "/etc/riak"}. +{platform_base_dir, "/usr/lib64/riak"}. +{platform_lib_dir, "/usr/lib64/riak/lib"}. +{platform_log_dir, "/var/log/riak"}. +{platform_gen_dir, "/var/lib/riak"}. + +{runner_script_dir, "/usr/lib64/riak/bin"}. +{runner_base_dir, "/usr/lib64/riak"}. +{runner_etc_dir, "/etc/riak"}. +{runner_log_dir, "/var/log/riak"}. +{runner_user, "riak"}. +{runner_lib_dir, "/usr/lib64/riak/lib"}. +{runner_patch_dir, "/usr/lib64/riak/lib/patches"}. +{pipe_dir, "/tmp/riak/"}. +{pid_dir, "/var/run/riak"}. +{package_replacement_line, ""}. +{package_conflicts_line, ""}. + +{cluster_manager_ip, "127.0.0.1"}. +{cluster_manager_port, 9080}. + +{web_ip, "127.0.0.1"}. +{web_port, 8098}. +{handoff_ip, "0.0.0.0"}. +{handoff_port, 8099}. +{pb_ip, "127.0.0.1"}. +{pb_port, 8087}. + +{storage_backend, "bitcask"}. + +{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. +{sasl_log_dir, "{{platform_log_dir}}/sasl"}. +{repl_data_root, "{{platform_data_dir}}/riak_repl/"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +%% +%% cuttlefish +%% +{cuttlefish, "on"}. +{cuttlefish_conf, "riak.conf"}. + +%% +%% yokozuna +%% +{yz_solr_port, 8093}. +{yz_solr_jmx_port, 8985}. + +%% lager +{console_log_default, file}. + +%% +%% etc/vm.args +%% +{node, "riak@127.0.0.1"}. diff --git a/rel/vars.config b/rel/vars.config index f4da0b9c1..6ac1c7e1a 100644 --- a/rel/vars.config +++ b/rel/vars.config @@ -7,6 +7,7 @@ {platform_etc_dir, "./etc"}. {platform_lib_dir, "./lib"}. {platform_log_dir, "./log"}. +{platform_gen_dir, "./releases/{{release_version}}"}. %% %% etc/app.config @@ -27,11 +28,6 @@ %% lager {console_log_default, file}. -%% Javascript VMs -{map_js_vms, 8}. -{reduce_js_vms, 6}. -{hook_js_vms, 2}. - %% %% etc/vm.args %%